Compare commits

...
Author SHA1 Message Date
0c16ec91b0 [feat]: connect Dreamverse creation settings to generation
Apply the backend-wiring changes beyond the UI uplift to
ds8/dreamversev2-dev for review and refactoring.

Source PR: hao-ai-lab/FastVideo#1854
Source range: 90d739a91892302edf37c4b23f807f402c83071d..8c5ee9b51cc3b75f4eba9cb3904fe08578c9dc9e
The 28-file patch is identical to that source range.

Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Aryan Kumar <aryan5v@users.noreply.github.com>
2026-09-15 17:16:15 -07:00
e57543b79d [feat]: Dreamverse creation studio UI uplift (#1853)
Co-authored-by: Cursor Agent <cursoragent@cursor.com>
Co-authored-by: Aryan Kumar <aryan5v@users.noreply.github.com>
2026-09-15 16:45:08 -07:00
42 changed files with 5709 additions and 514 deletions
+32 -3
View File
@@ -299,18 +299,47 @@ There are three related prompt paths in the current system:
## Initial Image And Segment Handling
The frontend currently sends `initial_image` as part of session init or
The frontend sends `initial_image` and, for first/last frame mode,
`last_frame_image` as part of `session_init_v2`, `project_init_v1`, or
`simple_generate`.
The server:
- validates and persists the image
- uses it only for segment 1 when present
- validates and persists the images
- uses `initial_image` only for segment 1 when present
- keeps continuation state for later segments in the GPU worker
This means the runtime, not the frontend, decides how segment 1 image
conditioning and later continuation conditioning are applied.
## Creation Studio Config
The lobby creation studio sends model, mode, aspect ratio, resolution, and
duration with session init. The server parses these fields into a per-session
creation config and echoes the resolved values back on `gpu_assigned` and
`ltx2_stream_start` as `creation_config`.
Incoming fields on `session_init_v2` and `project_init_v1`:
- `generation_mode`: `t2va`, `fl2va`, or `ref2va` (canonical upstream IDs from #1834)
- `model_id`: `fast-ltx2`, `fast-ltx23`, or `fast-h3`
- `aspect_ratio`: one of `21:9`, `16:9`, `4:3`, `1:1`, `3:4`, `9:16`
- `resolution`: one of `480p`, `720p`, `1080p`, `4k`
- `duration_sec`: `5`, `10`, or `15`
- `initial_image`: optional image payload for reference / first-frame modes
- `last_frame_image`: optional image payload for first/last frame mode
Echoed `creation_config` includes the resolved frame size,
`num_frames`, and `generation_segment_cap` derived from `duration_sec`.
Mode validation:
- `ref2va` requires `initial_image`
- `fl2va` requires both `initial_image` and `last_frame_image`
Per-step generation uses the resolved `frame_width`, `frame_height`, and
`num_frames` from the session creation config.
## Websocket Contract
The websocket is the main integration surface between UI and runtime.
@@ -0,0 +1,137 @@
from __future__ import annotations
from dataclasses import dataclass
from dreamverse.config import MODEL_REGISTRY
# Canonical upstream wire IDs. FL2VA is tracked in #1834 but not wired on Dreamverse
# streaming backends yet.
LTX_LOBBY_GENERATION_MODES = frozenset({"t2va", "ref2va"})
H3_LOBBY_GENERATION_MODES = frozenset({"t2va", "ref2va"})
LTX_LOBBY_ASPECT_RATIOS = frozenset({"21:9", "16:9", "4:3", "1:1", "3:4", "9:16"})
# Realtime FastLTX serving is validated through 1080p-class outputs; 4K is rejected
# until the runtime path is tested on Dreamverse GPUs.
LTX_LOBBY_RESOLUTIONS = frozenset({"480p", "720p", "1080p"})
# FastH3 serves a fixed 768x1344 (16:9-class) output; lobby resolution is nominal.
H3_LOBBY_ASPECT_RATIOS = frozenset({"16:9"})
H3_LOBBY_RESOLUTIONS = frozenset({"720p"})
LOBBY_DURATION_SEC = frozenset({5, 10, 15})
UNSUPPORTED_GENERATION_MODE_MESSAGES = {
"fl2va": "First/last frame mode (FL2VA) is not supported yet.",
}
@dataclass(frozen=True)
class ModelCreationCapabilities:
generation_modes: frozenset[str]
aspect_ratios: frozenset[str]
resolutions: frozenset[str]
duration_sec: frozenset[int]
unsupported_generation_modes: frozenset[str] = frozenset({"fl2va"})
def as_dict(self) -> dict[str, object]:
unsupported = {
mode: UNSUPPORTED_GENERATION_MODE_MESSAGES[mode]
for mode in sorted(self.unsupported_generation_modes)
if mode in UNSUPPORTED_GENERATION_MODE_MESSAGES
}
return {
"generation_modes": sorted(self.generation_modes),
"aspect_ratios": sorted(self.aspect_ratios),
"resolutions": sorted(self.resolutions),
"duration_sec": sorted(self.duration_sec),
"unsupported_generation_modes": unsupported,
"reference_assets": {
"mime_types": ["image/png", "image/jpeg", "image/webp"],
"max_bytes": 15 * 1024 * 1024,
},
}
LTX_MODEL_CREATION_CAPABILITIES = ModelCreationCapabilities(
generation_modes=LTX_LOBBY_GENERATION_MODES,
aspect_ratios=LTX_LOBBY_ASPECT_RATIOS,
resolutions=LTX_LOBBY_RESOLUTIONS,
duration_sec=LOBBY_DURATION_SEC,
)
H3_MODEL_CREATION_CAPABILITIES = ModelCreationCapabilities(
generation_modes=H3_LOBBY_GENERATION_MODES,
aspect_ratios=H3_LOBBY_ASPECT_RATIOS,
resolutions=H3_LOBBY_RESOLUTIONS,
duration_sec=LOBBY_DURATION_SEC,
)
MODEL_CREATION_CAPABILITIES: dict[str, ModelCreationCapabilities] = {
"fast-ltx2": LTX_MODEL_CREATION_CAPABILITIES,
"fast-ltx23": LTX_MODEL_CREATION_CAPABILITIES,
"fast-h3": H3_MODEL_CREATION_CAPABILITIES,
}
def capabilities_for_model(model_id: str) -> ModelCreationCapabilities:
if model_id not in MODEL_REGISTRY:
raise ValueError(f"Unknown model_id: {model_id}")
return MODEL_CREATION_CAPABILITIES.get(model_id, LTX_MODEL_CREATION_CAPABILITIES)
def lobby_capabilities_as_dict() -> dict[str, object]:
model_ids = sorted(MODEL_REGISTRY.keys())
models = {model_id: capabilities_for_model(model_id).as_dict() for model_id in model_ids}
union_modes: set[str] = set()
union_aspects: set[str] = set()
union_resolutions: set[str] = set()
union_durations: set[int] = set()
for caps in MODEL_CREATION_CAPABILITIES.values():
union_modes.update(caps.generation_modes)
union_aspects.update(caps.aspect_ratios)
union_resolutions.update(caps.resolutions)
union_durations.update(caps.duration_sec)
return {
"model_ids": model_ids,
"models": models,
"generation_modes": sorted(union_modes),
"aspect_ratios": sorted(union_aspects),
"resolutions": sorted(union_resolutions),
"duration_sec": sorted(union_durations),
"unsupported_generation_modes": dict(UNSUPPORTED_GENERATION_MODE_MESSAGES),
"reference_assets": {
"mime_types": ["image/png", "image/jpeg", "image/webp"],
"max_bytes": 15 * 1024 * 1024,
},
}
# Backward-compatible alias used in tests.
LOBBY_CREATION_CAPABILITIES = lobby_capabilities_as_dict()
def validate_lobby_creation_config(
*,
model_id: str,
generation_mode: str,
aspect_ratio: str,
resolution: str,
duration_sec: int,
) -> None:
if model_id not in MODEL_REGISTRY:
raise ValueError(f"Unknown model_id: {model_id}")
caps = capabilities_for_model(model_id)
if generation_mode in caps.unsupported_generation_modes:
raise ValueError(UNSUPPORTED_GENERATION_MODE_MESSAGES[generation_mode])
if generation_mode not in caps.generation_modes:
raise ValueError(f"Unsupported generation_mode: {generation_mode}")
if aspect_ratio not in caps.aspect_ratios:
raise ValueError(f"Unsupported aspect_ratio: {aspect_ratio}")
if resolution not in caps.resolutions:
raise ValueError(f"Unsupported resolution: {resolution}")
if duration_sec not in caps.duration_sec:
raise ValueError("duration_sec must be 5, 10, or 15.")
@@ -36,6 +36,10 @@ class GenerationBackend(Protocol):
segment_idx: int,
image_path: str | None,
reset_conditioning: bool,
*,
frame_width: int | None = None,
frame_height: int | None = None,
num_frames: int | None = None,
) -> StepResult:
...
@@ -80,6 +80,10 @@ class VideoGenerationWorker:
segment_idx: int,
image_path: str | None,
reset_conditioning: bool,
*,
frame_width: int | None = None,
frame_height: int | None = None,
num_frames: int | None = None,
) -> StepResult:
"""Generate one segment through the selected model backend."""
return self._require_backend().generate_step(
@@ -87,6 +91,9 @@ class VideoGenerationWorker:
segment_idx,
image_path,
reset_conditioning,
frame_width=frame_width,
frame_height=frame_height,
num_frames=num_frames,
)
def warmup(self, prompt: str) -> dict[str, float]:
+10
View File
@@ -189,6 +189,9 @@ def gpu_worker_process(
segment_idx,
image_path=payload.image_path,
reset_conditioning=payload.reset_conditioning,
frame_width=payload.frame_width,
frame_height=payload.frame_height,
num_frames=payload.num_frames,
)
head_trim_frames = step_result.head_trim_frames
head_trim_audio_frames = step_result.head_trim_audio_frames
@@ -753,6 +756,10 @@ class GPUSlot:
segment_idx: int = 1,
image_path: str | None = None,
reset_conditioning: bool = False,
*,
frame_width: int | None = None,
frame_height: int | None = None,
num_frames: int | None = None,
) -> dict[str, float]:
"""Execute a generation step for a specific user.
@@ -766,6 +773,9 @@ class GPUSlot:
segment_idx=segment_idx,
image_path=image_path,
reset_conditioning=bool(reset_conditioning),
frame_width=frame_width,
frame_height=frame_height,
num_frames=num_frames,
)
response = await self._send_command_tagged(Command(CommandType.USER_STEP, payload=payload, user_id=user_id),
timeout=1800.0)
@@ -454,6 +454,10 @@ class LTX2GenerationBackend:
segment_idx: int,
image_path: str | None,
reset_conditioning: bool,
*,
frame_width: int | None = None,
frame_height: int | None = None,
num_frames: int | None = None,
) -> StepResult:
"""Execute one generation step; snapshot state for the next segment."""
timings: dict = {}
@@ -464,9 +468,9 @@ class LTX2GenerationBackend:
prompt=prompt,
negative_prompt="",
save_video=False,
height=FRAME_HEIGHT,
width=FRAME_WIDTH,
num_frames=NUM_FRAMES,
height=frame_height or FRAME_HEIGHT,
width=frame_width or FRAME_WIDTH,
num_frames=num_frames or NUM_FRAMES,
fps=24,
num_inference_steps=NUM_INFERENCE_STEPS,
guidance_scale=1.0,
+2
View File
@@ -33,6 +33,7 @@ from dreamverse.routes.presets import (
prompt_config_router,
curated_presets_router,
)
from dreamverse.routes.creation import creation_router
from dreamverse.session.controller import SessionController
@@ -92,6 +93,7 @@ app.add_middleware(
app.include_router(build_health_router(lambda: runtime.gpu_pool))
app.include_router(internal_monitor_router)
app.include_router(prompt_config_router)
app.include_router(creation_router)
if DEVTOOLS_ENABLED:
app.include_router(curated_presets_router)
@@ -200,6 +200,10 @@ class MiniMaxH3GenerationBackend:
segment_idx: int,
image_path: str | None,
reset_conditioning: bool,
*,
frame_width: int | None = None,
frame_height: int | None = None,
num_frames: int | None = None,
) -> StepResult:
"""Generate one synchronized FastH3 segment and retain its last frame.
@@ -207,6 +211,7 @@ class MiniMaxH3GenerationBackend:
conditioned frame and its matching audio duration are trimmed before
streaming so adjacent segments do not duplicate media.
"""
del frame_width, frame_height, num_frames
if self.generator is None:
raise RuntimeError("FastH3 generator is not initialized.")
conditioning_image, uses_continuation = self._select_conditioning_image(
+65 -5
View File
@@ -31,6 +31,8 @@ from fastapi.staticfiles import StaticFiles
from dreamverse._deps import require_dreamverse_runtime_deps
from dreamverse.config import FRONTEND_STATIC_DIR_CANDIDATES, GENERATION_SEGMENT_CAP
from dreamverse.creation_capabilities import lobby_capabilities_as_dict
from dreamverse.session_creation_config import parse_session_creation_config, validate_generation_mode_assets
from dreamverse.session_init_image import cleanup_session_init_image, persist_session_init_image
LATENCY_MS = 200
@@ -225,6 +227,11 @@ async def prompt_system_config():
}
@app.get("/creation-capabilities")
async def creation_capabilities():
return lobby_capabilities_as_dict()
@app.get("/curated-presets")
async def curated_presets():
presets = [
@@ -290,6 +297,8 @@ async def websocket_endpoint(websocket: WebSocket):
send_lock = asyncio.Lock()
stop_event = asyncio.Event()
session_init_image = None
session_last_frame_image = None
session_creation_config = None
async def ws_send_json(payload: dict) -> None:
async with send_lock:
@@ -348,6 +357,7 @@ async def websocket_endpoint(websocket: WebSocket):
try:
session_init_image = persist_session_init_image(init_data.get("initial_image"))
session_last_frame_image = persist_session_init_image(init_data.get("last_frame_image"))
except ValueError as exc:
await ws_send_json({
"type": "error",
@@ -356,13 +366,31 @@ async def websocket_endpoint(websocket: WebSocket):
await websocket.close(code=1003, reason="Invalid initial image")
return
try:
session_creation_config = parse_session_creation_config(init_data)
validate_generation_mode_assets(
session_creation_config.generation_mode,
has_initial_image=session_init_image is not None,
has_last_frame_image=session_last_frame_image is not None,
)
except ValueError as exc:
await ws_send_json({
"type": "error",
"message": str(exc),
})
await websocket.close(code=1003, reason="Invalid creation config")
return
timeout_task = asyncio.create_task(session_timeout())
await ws_send_json({
gpu_assigned_payload: dict[str, object] = {
"type": "gpu_assigned",
"gpu_id": 0,
"session_timeout": SESSION_TIMEOUT_SECONDS,
})
}
if session_creation_config is not None:
gpu_assigned_payload["creation_config"] = session_creation_config.as_dict()
await ws_send_json(gpu_assigned_payload)
raw_prompt_queue: asyncio.Queue[PromptSubmission] = asyncio.Queue()
ready_prompt_queue: asyncio.Queue[ReadyPrompt] = asyncio.Queue()
@@ -391,8 +419,16 @@ async def websocket_endpoint(websocket: WebSocket):
if previous_session_image is not None:
cleanup_session_init_image(previous_session_image)
def replace_last_frame_image(last_frame_payload: object) -> None:
nonlocal session_last_frame_image
next_last_frame_image = persist_session_init_image(last_frame_payload)
previous_last_frame_image = session_last_frame_image
session_last_frame_image = next_last_frame_image
if previous_last_frame_image is not None:
cleanup_session_init_image(previous_last_frame_image)
async def send_stream_start(seed_reason: str) -> None:
await ws_send_json({
stream_start_payload: dict[str, object] = {
"type": "ltx2_stream_start",
"total_segments": len(curated_prompts),
"preset_id": preset_id,
@@ -400,8 +436,15 @@ async def websocket_endpoint(websocket: WebSocket):
"live_mode": True,
"loop_generation_enabled": loop_generation_enabled,
"loop_iteration": loop_iteration,
"generation_segment_cap": 0,
})
"generation_segment_cap": (
session_creation_config.generation_segment_cap
if session_creation_config is not None
else GENERATION_SEGMENT_CAP
),
}
if session_creation_config is not None:
stream_start_payload["creation_config"] = session_creation_config.as_dict()
await ws_send_json(stream_start_payload)
if seed_reason == "init":
await ws_send_json({
"type": "seed_prompts_updated",
@@ -509,6 +552,7 @@ async def websocket_endpoint(websocket: WebSocket):
nonlocal project_active
nonlocal project_stream_started
nonlocal pending_project_end
nonlocal session_creation_config
next_initial_rollout_prompt = str(payload.get("initial_rollout_prompt") or "").strip()
next_preset_id = str(payload.get("preset_id") or "").strip()
@@ -520,6 +564,21 @@ async def websocket_endpoint(websocket: WebSocket):
try:
replace_session_image(payload.get("initial_image"))
replace_last_frame_image(payload.get("last_frame_image"))
except ValueError as exc:
await ws_send_json({
"type": "error",
"message": str(exc),
})
return False
try:
session_creation_config = parse_session_creation_config(payload)
validate_generation_mode_assets(
session_creation_config.generation_mode,
has_initial_image=session_init_image is not None,
has_last_frame_image=session_last_frame_image is not None,
)
except ValueError as exc:
await ws_send_json({
"type": "error",
@@ -1182,6 +1241,7 @@ async def websocket_endpoint(websocket: WebSocket):
finally:
stop_event.set()
cleanup_session_init_image(session_init_image)
cleanup_session_init_image(session_last_frame_image)
for static_dir in FRONTEND_STATIC_DIR_CANDIDATES:
@@ -0,0 +1,14 @@
"""Creation studio capability routes."""
from __future__ import annotations
from fastapi import APIRouter
from dreamverse.creation_capabilities import lobby_capabilities_as_dict
creation_router = APIRouter(tags=["creation"])
@creation_router.get("/creation-capabilities")
async def creation_capabilities() -> dict[str, object]:
return lobby_capabilities_as_dict()
+105 -44
View File
@@ -27,6 +27,7 @@ from typing import TYPE_CHECKING
from fastapi import WebSocket, WebSocketDisconnect
from dreamverse.gpu_pool import GPUSlot
from dreamverse.session_init_image import cleanup_session_init_image, persist_session_init_image
from dreamverse.session_creation_config import parse_session_creation_config, validate_generation_mode_assets
from dreamverse.worker_ipc import MediaChunk, MediaComplete, MediaInit
from dreamverse.config import (
@@ -156,6 +157,9 @@ class SessionController:
prompt_worker_task: asyncio.Task | None = None
rewrite_seed_prompts_task: asyncio.Task | None = None
session_init_image = None
session_last_frame_image = None
session_creation_config = None
session_generation_segment_cap = GENERATION_SEGMENT_CAP
async def session_timeout():
"""Close the session after timeout."""
@@ -237,6 +241,7 @@ class SessionController:
try:
session_init_image = persist_session_init_image(init_data.get("initial_image"))
session_last_frame_image = persist_session_init_image(init_data.get("last_frame_image"))
except ValueError as exc:
await ws_send_json({
"type": "error",
@@ -245,6 +250,22 @@ class SessionController:
await websocket.close(code=1003, reason="Invalid initial image")
return
try:
session_creation_config = parse_session_creation_config(init_data)
session_generation_segment_cap = session_creation_config.generation_segment_cap
validate_generation_mode_assets(
session_creation_config.generation_mode,
has_initial_image=session_init_image is not None,
has_last_frame_image=session_last_frame_image is not None,
)
except ValueError as exc:
await ws_send_json({
"type": "error",
"message": str(exc),
})
await websocket.close(code=1003, reason="Invalid creation config")
return
if preset_id:
print(f"Client {client_id[:8]} selected preset: {preset_id} "
f"label={preset_label or '(unset)'} "
@@ -256,6 +277,16 @@ class SessionController:
if session_init_image is not None:
print(f"Client {client_id[:8]} uploaded initial image: "
f"{session_init_image.display_name}")
if session_last_frame_image is not None:
print(f"Client {client_id[:8]} uploaded last frame image: "
f"{session_last_frame_image.display_name}")
if session_creation_config is not None:
print(f"Client {client_id[:8]} creation config: "
f"model={session_creation_config.model_id}, "
f"mode={session_creation_config.generation_mode}, "
f"size={session_creation_config.frame_width}x{session_creation_config.frame_height}, "
f"duration={session_creation_config.duration_sec}s, "
f"segment_cap={session_creation_config.generation_segment_cap}")
# Acquire a GPU slot.
gpu_id, slot = await self.gpu_pool.acquire(client_id, websocket)
@@ -264,14 +295,20 @@ class SessionController:
timeout_task = asyncio.create_task(session_timeout())
# Join the engine on this GPU.
await slot.join_user(client_id, model_id=ACTIVE_MODEL_ID)
await slot.join_user(
client_id,
model_id=session_creation_config.model_id if session_creation_config is not None else ACTIVE_MODEL_ID,
)
# Notify client they're connected to a GPU.
await ws_send_json({
gpu_assigned_payload: dict[str, object] = {
"type": "gpu_assigned",
"gpu_id": gpu_id,
"session_timeout": SESSION_TIMEOUT_SECONDS,
})
}
if session_creation_config is not None:
gpu_assigned_payload["creation_config"] = session_creation_config.as_dict()
await ws_send_json(gpu_assigned_payload)
await log_event(
"gpu_assigned",
{
@@ -315,6 +352,14 @@ class SessionController:
if previous_session_init_image is not None:
cleanup_session_init_image(previous_session_init_image)
def replace_last_frame_image(last_frame_payload: object) -> None:
nonlocal session_last_frame_image
next_last_frame_image = persist_session_init_image(last_frame_payload)
previous_last_frame_image = session_last_frame_image
session_last_frame_image = next_last_frame_image
if previous_last_frame_image is not None:
cleanup_session_init_image(previous_last_frame_image)
async def schedule_simple_generate_request(payload: dict[str, object]) -> None:
nonlocal preset_id
nonlocal preset_label
@@ -452,6 +497,8 @@ class SessionController:
nonlocal project_active
nonlocal project_stream_started
nonlocal pending_project_end
nonlocal session_creation_config
nonlocal session_generation_segment_cap
next_initial_rollout_prompt = str(payload.get("initial_rollout_prompt") or "").strip()
next_enhancement_enabled = bool(payload.get("enhancement_enabled", True))
@@ -498,6 +545,22 @@ class SessionController:
try:
replace_session_init_image(payload.get("initial_image"))
replace_last_frame_image(payload.get("last_frame_image"))
except ValueError as exc:
await ws_send_json({
"type": "error",
"message": str(exc),
})
return False
try:
session_creation_config = parse_session_creation_config(payload)
session_generation_segment_cap = session_creation_config.generation_segment_cap
validate_generation_mode_assets(
session_creation_config.generation_mode,
has_initial_image=session_init_image is not None,
has_last_frame_image=session_last_frame_image is not None,
)
except ValueError as exc:
await ws_send_json({
"type": "error",
@@ -941,7 +1004,7 @@ class SessionController:
"segment_cap":
_resolve_generation_segment_cap(
single_clip_mode=single_clip_mode,
cap=GENERATION_SEGMENT_CAP,
cap=session_generation_segment_cap,
),
})
continue
@@ -1281,27 +1344,22 @@ class SessionController:
))
else:
project_stream_started = True
await ws_send_json({
"type":
"ltx2_stream_start",
"total_segments":
len(curated_prompts),
"preset_id":
preset_id,
"stream_mode":
"av_fmp4",
"live_mode":
True,
"loop_generation_enabled":
loop_generation_enabled,
"loop_iteration":
loop_iteration,
"generation_segment_cap":
_resolve_generation_segment_cap(
stream_start_payload: dict[str, object] = {
"type": "ltx2_stream_start",
"total_segments": len(curated_prompts),
"preset_id": preset_id,
"stream_mode": "av_fmp4",
"live_mode": True,
"loop_generation_enabled": loop_generation_enabled,
"loop_iteration": loop_iteration,
"generation_segment_cap": _resolve_generation_segment_cap(
single_clip_mode=single_clip_mode,
cap=GENERATION_SEGMENT_CAP,
cap=session_generation_segment_cap,
),
})
}
if session_creation_config is not None:
stream_start_payload["creation_config"] = session_creation_config.as_dict()
await ws_send_json(stream_start_payload)
await ws_send_json({
"type": "seed_prompts_updated",
"prompts": seed_prompt_memory,
@@ -1340,27 +1398,22 @@ class SessionController:
loop_iteration += 1
project_stream_started = True
await ws_send_json({
"type":
"ltx2_stream_start",
"total_segments":
len(curated_prompts),
"preset_id":
preset_id,
"stream_mode":
"av_fmp4",
"live_mode":
True,
"loop_generation_enabled":
loop_generation_enabled,
"loop_iteration":
loop_iteration,
"generation_segment_cap":
_resolve_generation_segment_cap(
restart_stream_payload: dict[str, object] = {
"type": "ltx2_stream_start",
"total_segments": len(curated_prompts),
"preset_id": preset_id,
"stream_mode": "av_fmp4",
"live_mode": True,
"loop_generation_enabled": loop_generation_enabled,
"loop_iteration": loop_iteration,
"generation_segment_cap": _resolve_generation_segment_cap(
single_clip_mode=single_clip_mode,
cap=GENERATION_SEGMENT_CAP,
cap=session_generation_segment_cap,
),
})
}
if session_creation_config is not None:
restart_stream_payload["creation_config"] = session_creation_config.as_dict()
await ws_send_json(restart_stream_payload)
if nonlocal_reason == "loop_restart":
await ws_send_json({
"type": "loop_restarted",
@@ -1389,13 +1442,14 @@ class SessionController:
pending_simple_prompt_submission = None
if (not single_clip_mode and not generation_cap_blocked and not rollout_waiting_for_rewrite
and GENERATION_SEGMENT_CAP > 0 and generated_segment_count >= GENERATION_SEGMENT_CAP):
and session_generation_segment_cap > 0
and generated_segment_count >= session_generation_segment_cap):
loop_generation_enabled = False
rollout_waiting_for_rewrite = True
_main_print(
"INFO",
f"Segment cap reached for client {client_id[:8]} "
f"(cap_segments={GENERATION_SEGMENT_CAP}, "
f"(cap_segments={session_generation_segment_cap}, "
f"generated_segments={generated_segment_count}); "
"waiting for rollout rewrite",
)
@@ -1620,6 +1674,9 @@ class SessionController:
pending_reset_conditioning = False
step_image_path = (str(session_init_image.file_path)
if segment_idx == 1 and session_init_image is not None else None)
step_frame_width = session_creation_config.frame_width if session_creation_config is not None else None
step_frame_height = session_creation_config.frame_height if session_creation_config is not None else None
step_num_frames = session_creation_config.num_frames if session_creation_config is not None else None
step_task = asyncio.create_task(
slot.user_step(
client_id,
@@ -1627,6 +1684,9 @@ class SessionController:
segment_idx=segment_idx,
image_path=step_image_path,
reset_conditioning=step_reset_conditioning,
frame_width=step_frame_width,
frame_height=step_frame_height,
num_frames=step_num_frames,
))
segment_generation_active = True
try:
@@ -1808,3 +1868,4 @@ class SessionController:
await self.gpu_pool.release(client_id)
finally:
cleanup_session_init_image(session_init_image)
cleanup_session_init_image(session_last_frame_image)
@@ -0,0 +1,140 @@
from __future__ import annotations
from dataclasses import dataclass
from dreamverse.config import FRAME_HEIGHT, FRAME_WIDTH, GENERATION_SEGMENT_CAP, MODEL_REGISTRY, NUM_FRAMES
from dreamverse.creation_capabilities import validate_lobby_creation_config
LTX_LOBBY_MODEL_IDS = frozenset(MODEL_REGISTRY.keys())
SUPPORTED_GENERATION_MODES = frozenset({"t2va", "fl2va", "ref2va"})
SUPPORTED_ASPECT_RATIOS = frozenset({"21:9", "16:9", "4:3", "1:1", "3:4", "9:16"})
SUPPORTED_RESOLUTIONS = frozenset({"480p", "720p", "1080p", "4k"})
SEGMENT_DURATION_SEC = 5
@dataclass(frozen=True)
class SessionCreationConfig:
model_id: str
generation_mode: str
aspect_ratio: str
resolution: str
duration_sec: int
frame_width: int
frame_height: int
num_frames: int
generation_segment_cap: int
def as_dict(self) -> dict[str, object]:
return {
"model_id": self.model_id,
"generation_mode": self.generation_mode,
"aspect_ratio": self.aspect_ratio,
"resolution": self.resolution,
"duration_sec": self.duration_sec,
"frame_width": self.frame_width,
"frame_height": self.frame_height,
"num_frames": self.num_frames,
"generation_segment_cap": self.generation_segment_cap,
}
def _round_to_multiple(value: float, multiple: int = 32) -> int:
rounded = int(round(value / multiple)) * multiple
return max(multiple, rounded)
def _resolution_base(resolution: str) -> int:
return {
"480p": 480,
"720p": 720,
"1080p": 1080,
"4k": 2160,
}.get(resolution, 720)
def resolve_frame_size(aspect_ratio: str, resolution: str) -> tuple[int, int]:
if aspect_ratio == "16:9" and resolution == "1080p":
return FRAME_WIDTH, FRAME_HEIGHT
base = _resolution_base(resolution)
width_ratio, height_ratio = {
"21:9": (21, 9),
"16:9": (16, 9),
"4:3": (4, 3),
"1:1": (1, 1),
"3:4": (3, 4),
"9:16": (9, 16),
}.get(aspect_ratio, (16, 9))
if width_ratio >= height_ratio:
height = _round_to_multiple(base)
width = _round_to_multiple(height * width_ratio / height_ratio)
else:
width = _round_to_multiple(base)
height = _round_to_multiple(width * height_ratio / width_ratio)
return width, height
def duration_sec_to_segment_cap(duration_sec: int, *, global_cap: int = GENERATION_SEGMENT_CAP) -> int:
requested = max(1, int(round(duration_sec / SEGMENT_DURATION_SEC + 0.0001)))
if global_cap <= 0:
return requested
return max(1, min(requested, global_cap))
def parse_session_creation_config(payload: dict[str, object]) -> SessionCreationConfig:
raw_model_id = str(payload.get("model_id") or "").strip()
model_id = raw_model_id if raw_model_id in LTX_LOBBY_MODEL_IDS else "fast-ltx23"
generation_mode = str(payload.get("generation_mode") or "t2va").strip()
if generation_mode not in SUPPORTED_GENERATION_MODES:
raise ValueError(f"Unsupported generation_mode: {generation_mode}")
aspect_ratio = str(payload.get("aspect_ratio") or "16:9").strip()
if aspect_ratio not in SUPPORTED_ASPECT_RATIOS:
raise ValueError(f"Unsupported aspect_ratio: {aspect_ratio}")
resolution = str(payload.get("resolution") or "720p").strip()
if resolution not in SUPPORTED_RESOLUTIONS:
raise ValueError(f"Unsupported resolution: {resolution}")
try:
duration_sec = int(payload.get("duration_sec") or SEGMENT_DURATION_SEC)
except (TypeError, ValueError) as exc:
raise ValueError("duration_sec must be an integer.") from exc
if duration_sec not in {5, 10, 15}:
raise ValueError("duration_sec must be 5, 10, or 15.")
validate_lobby_creation_config(
model_id=model_id,
generation_mode=generation_mode,
aspect_ratio=aspect_ratio,
resolution=resolution,
duration_sec=duration_sec,
)
if model_id not in MODEL_REGISTRY:
raise ValueError(f"Unsupported model_id: {model_id}")
frame_width, frame_height = resolve_frame_size(aspect_ratio, resolution)
return SessionCreationConfig(
model_id=model_id,
generation_mode=generation_mode,
aspect_ratio=aspect_ratio,
resolution=resolution,
duration_sec=duration_sec,
frame_width=frame_width,
frame_height=frame_height,
num_frames=NUM_FRAMES,
generation_segment_cap=duration_sec_to_segment_cap(duration_sec),
)
def validate_generation_mode_assets(
generation_mode: str,
*,
has_initial_image: bool,
has_last_frame_image: bool,
) -> None:
if generation_mode == "ref2va" and not has_initial_image:
raise ValueError("Ref2VA mode requires a reference image.")
@@ -0,0 +1,74 @@
import pytest
from dreamverse.creation_capabilities import (
capabilities_for_model,
lobby_capabilities_as_dict,
validate_lobby_creation_config,
)
def test_lobby_capabilities_include_all_registry_models():
caps = lobby_capabilities_as_dict()
assert set(caps["model_ids"]) == {"fast-ltx2", "fast-ltx23", "fast-h3"}
assert "fl2va" not in caps["generation_modes"]
assert "4k" not in caps["resolutions"]
def test_fast_h3_capabilities_use_fixed_geometry():
h3_caps = capabilities_for_model("fast-h3")
assert h3_caps.generation_modes == frozenset({"t2va", "ref2va"})
assert h3_caps.aspect_ratios == frozenset({"16:9"})
assert h3_caps.resolutions == frozenset({"720p"})
def test_validate_lobby_creation_config_accepts_supported_t2va():
validate_lobby_creation_config(
model_id="fast-ltx23",
generation_mode="t2va",
aspect_ratio="16:9",
resolution="1080p",
duration_sec=5,
)
def test_validate_lobby_creation_config_accepts_fast_h3():
validate_lobby_creation_config(
model_id="fast-h3",
generation_mode="ref2va",
aspect_ratio="16:9",
resolution="720p",
duration_sec=10,
)
def test_validate_lobby_creation_config_rejects_fl2va():
with pytest.raises(ValueError, match="FL2VA"):
validate_lobby_creation_config(
model_id="fast-ltx23",
generation_mode="fl2va",
aspect_ratio="16:9",
resolution="720p",
duration_sec=5,
)
def test_validate_lobby_creation_config_rejects_4k():
with pytest.raises(ValueError, match="Unsupported resolution"):
validate_lobby_creation_config(
model_id="fast-ltx2",
generation_mode="t2va",
aspect_ratio="16:9",
resolution="4k",
duration_sec=10,
)
def test_validate_lobby_creation_config_rejects_invalid_h3_aspect():
with pytest.raises(ValueError, match="Unsupported aspect_ratio"):
validate_lobby_creation_config(
model_id="fast-h3",
generation_mode="t2va",
aspect_ratio="9:16",
resolution="720p",
duration_sec=5,
)
@@ -0,0 +1,89 @@
import pytest
from dreamverse.session_creation_config import (
duration_sec_to_segment_cap,
parse_session_creation_config,
resolve_frame_size,
validate_generation_mode_assets,
)
def test_parse_session_creation_config_defaults():
config = parse_session_creation_config({})
assert config.model_id == "fast-ltx23"
assert config.generation_mode == "t2va"
assert config.aspect_ratio == "16:9"
assert config.resolution == "720p"
assert config.duration_sec == 5
assert config.generation_segment_cap == 1
def test_parse_session_creation_config_maps_duration_to_segment_cap():
config = parse_session_creation_config(
{
"model_id": "fast-ltx2",
"generation_mode": "ref2va",
"aspect_ratio": "9:16",
"resolution": "480p",
"duration_sec": 15,
},
)
assert config.model_id == "fast-ltx2"
assert config.generation_mode == "ref2va"
assert config.generation_segment_cap == 3
assert config.frame_width >= 480
assert config.frame_height >= 480
def test_resolve_frame_size_uses_model_default_for_1080p_landscape():
width, height = resolve_frame_size("16:9", "1080p")
assert (width, height) == (1920, 1088)
def test_duration_sec_to_segment_cap_respects_global_cap():
assert duration_sec_to_segment_cap(15, global_cap=2) == 2
def test_parse_session_creation_config_accepts_fast_h3():
config = parse_session_creation_config(
{
"model_id": "fast-h3",
"generation_mode": "t2va",
"aspect_ratio": "16:9",
"resolution": "720p",
"duration_sec": 10,
},
)
assert config.model_id == "fast-h3"
assert config.generation_mode == "t2va"
assert config.generation_segment_cap == 2
def test_parse_session_creation_config_rejects_fl2va():
with pytest.raises(ValueError, match="FL2VA"):
parse_session_creation_config(
{
"generation_mode": "fl2va",
"aspect_ratio": "16:9",
"resolution": "720p",
"duration_sec": 5,
},
)
def test_parse_session_creation_config_rejects_4k():
with pytest.raises(ValueError, match="Unsupported resolution"):
parse_session_creation_config(
{
"generation_mode": "t2va",
"aspect_ratio": "16:9",
"resolution": "4k",
"duration_sec": 5,
},
)
def test_validate_generation_mode_assets():
validate_generation_mode_assets("t2va", has_initial_image=False, has_last_frame_image=False)
with pytest.raises(ValueError, match="Ref2VA"):
validate_generation_mode_assets("ref2va", has_initial_image=False, has_last_frame_image=False)
+3
View File
@@ -147,6 +147,9 @@ class UserStepPayload:
segment_idx: int
image_path: str | None
reset_conditioning: bool
frame_width: int | None = None
frame_height: int | None = None
num_frames: int | None = None
@dataclass(frozen=True)
+4 -12
View File
@@ -17,25 +17,17 @@ test.describe('frontend shell', () => {
});
});
test('composer hydrates with curated preset cards', async ({ page }) => {
test('composer hydrates with creation studio controls', async ({ page }) => {
await page.goto('/');
// The Continuation prompt textarea + Generate button render once
// the FE has hydrated against the public-FastVideo-backed
// dreamverse-server. Their presence proves the integration handshake
// (CORS, /curated-presets, /prompt-system-config) completed.
const continuation = page.getByLabel('Continuation prompt');
await expect(continuation).toBeVisible({ timeout: 30_000 });
const generate = page.getByRole('button', { name: /^generate$/i });
await expect(generate).toBeVisible({ timeout: 30_000 });
// Curated presets render as buttons; verify at least one is
// available — that's the only way the user can populate the
// Continuation textarea in the default composer.
const presetCard = page.getByRole('button', {
name: /LEGO Stormtroopers|Clay Stop-Motion|Boy & Dog|School Prank|Gamer Gets Banned|Small Town Oil Strike|Grandpa's Wing Costume/i,
}).first();
await expect(presetCard).toBeVisible({ timeout: 30_000 });
await expect(page.getByText('Direct scenes in seconds')).toBeVisible({ timeout: 30_000 });
await expect(page.getByRole('button', { name: /FastLTX/i }).first()).toBeVisible({ timeout: 30_000 });
await expect(continuation).toHaveAttribute('placeholder', /Describe your video or mention elements/i);
});
});
+4
View File
@@ -42,6 +42,10 @@ const nextConfig: NextConfig = {
source: '/prompt-system-config',
destination: `${backendUrl}/prompt-system-config`,
},
{
source: '/creation-capabilities',
destination: `${backendUrl}/creation-capabilities`,
},
{
source: '/curated-presets',
destination: `${backendUrl}/curated-presets`,
+2262 -113
View File
File diff suppressed because it is too large Load Diff
+4
View File
@@ -27,11 +27,15 @@
"@radix-ui/react-accordion": "^1.2.12",
"@radix-ui/react-checkbox": "^1.3.3",
"@radix-ui/react-collapsible": "^1.1.12",
"@radix-ui/react-dropdown-menu": "^2.1.24",
"@radix-ui/react-label": "^2.1.8",
"@radix-ui/react-popover": "^1.1.23",
"@radix-ui/react-scroll-area": "^1.2.10",
"@radix-ui/react-select": "^2.2.6",
"@radix-ui/react-separator": "^1.1.8",
"@radix-ui/react-slider": "^1.4.7",
"@radix-ui/react-slot": "^1.2.4",
"@radix-ui/react-tabs": "^1.1.21",
"class-variance-authority": "^0.7.1",
"clsx": "^2.1.1",
"framer-motion": "^12.36.0",
+26 -1
View File
@@ -2,6 +2,7 @@
@import "tailwindcss";
@custom-variant dark (&:is(.dark *));
@custom-variant hover-capable (@media (hover: hover) and (pointer: fine));
:root {
color-scheme: light;
@@ -201,7 +202,7 @@ summary::-webkit-details-marker {
@apply border-border;
}
body {
@apply bg-background text-foreground;
@apply bg-background text-foreground antialiased;
}
}
@@ -220,6 +221,14 @@ summary::-webkit-details-marker {
.stroke-dash-anim {
animation: stroke-dash-animation 2s linear infinite;
}
.text-pretty {
text-wrap: pretty;
}
.text-balance {
text-wrap: balance;
}
}
@keyframes stroke-dash-animation {
@@ -287,6 +296,22 @@ html.theme-transition *::after {
/* —————————————— CUSTOM TAILWIND —————————————— */
@layer components {
.studio-control {
@apply transition-[border-color,background-color,box-shadow,color,transform] duration-150 focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent-blue/40;
}
.studio-control-press {
@apply active:scale-[0.96] motion-reduce:active:scale-100;
}
.studio-hover-surface {
@apply hover-capable:hover:border-border hover-capable:hover:bg-accent/50;
}
.studio-media-outline {
@apply outline outline-1 -outline-offset-1 outline-black/10 dark:outline-white/10;
}
.debug {
@apply border border-rose-500;
}
+300 -136
View File
@@ -1,10 +1,20 @@
"use client";
import { Fragment, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState } from "react";
import { Fragment, useCallback, useEffect, useLayoutEffect, useMemo, useRef, useState, type Dispatch, type SetStateAction } from "react";
import { AnimatePresence, motion } from "framer-motion";
import { Download, Share2 } from "lucide-react";
import DevtoolsShell from "@/components/devtools/DevtoolsShell";
import MonitorPage from "@/components/MonitorPage";
import ChatBar from "@/components/ChatBar";
import CreationStudio from "@/components/creation/CreationStudio";
import {
buildMentionOptions,
type AspectRatioId,
type CreationModeId,
type CreationModelId,
type ResolutionId,
} from "@/lib/creationConfig";
import { toGenerationMode } from "@/lib/generationMode";
import type { SessionCreationConfig } from "@/components/creation/SessionCreationConfigPills";
import SessionTimeoutModal from "@/components/SessionTimeoutModal";
import Sidebar from "@/components/Sidebar";
import Header from "@/components/Header";
@@ -22,6 +32,18 @@ import {
buildRewritePromptWindowSnapshotFromPrompts,
normalizePromptWindowSnapshot,
} from "@/lib/prompts/promptWindowSnapshot";
import {
DEFAULT_LOBBY_CAPABILITIES_BUNDLE,
clampLobbySelectionToCapabilities,
parseLobbyCapabilitiesBundle,
resolveModelCapabilities,
validateLobbyCreationSelection,
type LobbyCapabilitiesBundle,
} from "@/lib/creationCapabilities";
import {
buildCreationInitPayload,
parseEchoedCreationConfig,
} from "@/lib/creationPayload";
import rawPresets from "@/lib/storyPresetsData";
import { cn } from "@/lib/utils";
import { createWebSocketConnection, detachAndCloseWebSocket } from "@/lib/ws/client";
@@ -78,98 +100,6 @@ function yieldToEventLoop(): Promise<void> {
return new Promise((r) => setTimeout(r, 0));
}
const HERO_WAVE_LIGHT = ["#2A4A98", "#4878E5", "#6FA0F2", "#B0BCC8", "#E8D99E", "#D8C844", "#C2A620"];
const HERO_WAVE_DARK = ["#143468", "#1E58B8", "#3892F0", "#80B8E8", "#B8D0EA", "#E2D498", "#DABB50"];
const HERO_TEXT = "Direct scenes in seconds";
function HeroTagline() {
const ref = useRef<HTMLHeadingElement>(null);
useEffect(() => {
const el = ref.current;
if (!el) return;
let rafId = 0;
function play() {
const chars = el!.querySelectorAll<HTMLSpanElement>("[data-char]");
if (!chars.length) return;
cancelAnimationFrame(rafId);
const isDark = document.documentElement.classList.contains("dark");
const colors = isDark ? HERO_WAVE_DARK : HERO_WAVE_LIGHT;
const waveLen = 10;
const total = chars.length + waveLen;
const duration = 1200;
const maxBlur = 3.5;
const start = performance.now();
function tick() {
const t = Math.min((performance.now() - start) / duration, 1);
const pos = t * total;
chars.forEach((ch, i) => {
const rel = pos - i;
if (rel >= 0 && rel < waveLen) {
const norm = rel / waveLen;
const ci = Math.floor(norm * colors.length);
ch.style.color = colors[Math.min(colors.length - 1, ci)];
let blur = 0;
if (norm < 0.25) {
blur = maxBlur * (1 - norm / 0.25);
} else if (norm > 0.75) {
blur = maxBlur * ((norm - 0.75) / 0.25);
}
ch.style.filter = blur > 0.1 ? `blur(${blur.toFixed(1)}px)` : "";
} else {
ch.style.color = "";
ch.style.filter = "";
}
});
if (t < 1) {
rafId = requestAnimationFrame(tick);
} else {
chars.forEach((ch) => {
ch.style.color = "";
ch.style.filter = "";
});
}
}
rafId = requestAnimationFrame(tick);
}
const initialDelay = setTimeout(play, 400);
const interval = setInterval(play, 5000);
return () => {
clearTimeout(initialDelay);
clearInterval(interval);
cancelAnimationFrame(rafId);
};
}, []);
return (
<h1 ref={ref} className="text-center text-3xl font-medium text-[#343537] dark:text-[#FAFAFB] sm:text-4xl">
{HERO_TEXT.split(" ").map((word, wi) => (
<Fragment key={wi}>
{wi > 0 && (
<span data-char className="transition-[color,filter] duration-150">
{" "}
</span>
)}
<span className="inline-flex">
{word.split("").map((char, ci) => (
<span key={ci} data-char className="inline-block transition-[color,filter] duration-150">
{char}
</span>
))}
</span>
</Fragment>
))}
</h1>
);
}
export default function Page() {
const storesRef = useRef<PageStores | null>(null);
if (!storesRef.current) {
@@ -323,8 +253,33 @@ export default function Page() {
const [ttffValueMs, setTtffValueMs] = useState<number | null>(null);
const ttffIntervalRef = useRef<ReturnType<typeof setInterval> | null>(null);
const pendingInitialPromptRef = useRef("");
const referenceFileRef = useRef<File | null>(null);
const firstFrameFileRef = useRef<File | null>(null);
const lastFrameFileRef = useRef<File | null>(null);
const lastArchivedReplayKeyRef = useRef("");
const [sidebarOpen, setSidebarOpen] = useState(false);
const [creationModelId, setCreationModelId] = useState<CreationModelId>("fast-ltx23");
const [creationModeId, setCreationModeId] = useState<CreationModeId>("t2v");
const [creationAspectRatio, setCreationAspectRatio] = useState<AspectRatioId>("16:9");
const [creationResolution, setCreationResolution] = useState<ResolutionId>("720p");
const [creationDurationSec, setCreationDurationSec] = useState(5);
const [lobbyCapabilitiesBundle, setLobbyCapabilitiesBundle] = useState<LobbyCapabilitiesBundle>(
DEFAULT_LOBBY_CAPABILITIES_BUNDLE,
);
const activeModelCapabilities = useMemo(
() => resolveModelCapabilities(lobbyCapabilitiesBundle, creationModelId),
[lobbyCapabilitiesBundle, creationModelId],
);
const [sessionCreationConfig, setSessionCreationConfig] = useState<SessionCreationConfig>({
modelId: "fast-ltx23",
modeId: "t2v",
aspectRatio: "16:9",
resolution: "720p",
durationSec: 5,
});
const [referencePreviewUrl, setReferencePreviewUrl] = useState<string | null>(null);
const [firstFramePreviewUrl, setFirstFramePreviewUrl] = useState<string | null>(null);
const [lastFramePreviewUrl, setLastFramePreviewUrl] = useState<string | null>(null);
const [currentThumbnail, setCurrentThumbnail] = useState<string | null>(null);
const currentProjectIdRef = useRef("");
const currentProjectCreatedAtRef = useRef(0);
@@ -345,6 +300,57 @@ export default function Page() {
setIsMobileShareCapable(typeof navigator.canShare === "function" && window.matchMedia("(pointer: coarse)").matches);
}, []);
useEffect(() => {
return () => {
if (referencePreviewUrl) {
URL.revokeObjectURL(referencePreviewUrl);
}
if (firstFramePreviewUrl) {
URL.revokeObjectURL(firstFramePreviewUrl);
}
if (lastFramePreviewUrl) {
URL.revokeObjectURL(lastFramePreviewUrl);
}
};
}, [referencePreviewUrl, firstFramePreviewUrl, lastFramePreviewUrl]);
function setPreviewUrl(setter: Dispatch<SetStateAction<string | null>>, file: File | null) {
setter((current) => {
if (current) URL.revokeObjectURL(current);
return file ? URL.createObjectURL(file) : null;
});
}
function handleReferenceSelect(file: File | null) {
referenceFileRef.current = file;
setPreviewUrl(setReferencePreviewUrl, file);
}
function handleFirstFrameSelect(file: File | null) {
firstFrameFileRef.current = file;
setPreviewUrl(setFirstFramePreviewUrl, file);
}
function handleLastFrameSelect(file: File | null) {
lastFrameFileRef.current = file;
setPreviewUrl(setLastFramePreviewUrl, file);
}
const mentionOptions = useMemo(() => buildMentionOptions(storyPresets as Array<{ id?: string; label?: string; description?: string }>), [storyPresets]);
const lobbyStoryPresets = useMemo(
() =>
(storyPresets as Array<{ id?: string; label?: string; description?: string; segment_prompts?: unknown }>)
.filter((preset) => typeof preset.id === "string" && typeof preset.label === "string")
.map((preset) => ({
id: String(preset.id),
label: String(preset.label),
description: typeof preset.description === "string" ? preset.description : undefined,
segmentCount: Array.isArray(preset.segment_prompts) ? preset.segment_prompts.length : undefined,
})),
[storyPresets],
);
const videoElRef = useRef<HTMLVideoElement | null>(null);
const archivedPlaybackElRef = useRef<HTMLVideoElement | null>(null);
const viewingModePlaybackStateRef = useRef<{
@@ -524,6 +530,65 @@ export default function Page() {
setRuntimeReady(true);
}, []);
function applyLobbyCapabilitiesBundle(bundle: LobbyCapabilitiesBundle) {
setLobbyCapabilitiesBundle(bundle);
const clamped = clampLobbySelectionToCapabilities({
capabilities: resolveModelCapabilities(bundle, creationModelId),
modelId: creationModelId,
modeId: creationModeId,
aspectRatio: creationAspectRatio,
resolution: creationResolution,
durationSec: creationDurationSec,
});
setCreationModelId(clamped.modelId);
setCreationModeId(clamped.modeId);
setCreationAspectRatio(clamped.aspectRatio);
setCreationResolution(clamped.resolution);
setCreationDurationSec(clamped.durationSec);
}
function handleCreationModelChange(modelId: CreationModelId) {
const clamped = clampLobbySelectionToCapabilities({
capabilities: resolveModelCapabilities(lobbyCapabilitiesBundle, modelId),
modelId,
modeId: creationModeId,
aspectRatio: creationAspectRatio,
resolution: creationResolution,
durationSec: creationDurationSec,
});
setCreationModelId(clamped.modelId);
setCreationModeId(clamped.modeId);
setCreationAspectRatio(clamped.aspectRatio);
setCreationResolution(clamped.resolution);
setCreationDurationSec(clamped.durationSec);
}
useEffect(() => {
if (!runtimeReady) return;
let cancelled = false;
void fetch("/creation-capabilities", {
headers: { Accept: "application/json" },
cache: "no-store",
})
.then(async (response) => {
if (!response.ok) return DEFAULT_LOBBY_CAPABILITIES_BUNDLE;
return parseLobbyCapabilitiesBundle(await response.json());
})
.then((bundle) => {
if (!cancelled) {
applyLobbyCapabilitiesBundle(bundle);
}
})
.catch(() => {
if (!cancelled) {
applyLobbyCapabilitiesBundle(DEFAULT_LOBBY_CAPABILITIES_BUNDLE);
}
});
return () => {
cancelled = true;
};
}, [runtimeReady]);
useEffect(() => {
if (!runtimeReady || initializedRef.current) return;
initializedRef.current = true;
@@ -1759,33 +1824,44 @@ export default function Page() {
resetPlaybackState();
}
function buildProjectInitPayload(type: "session_init_v2" | "project_init_v1") {
async function buildProjectInitPayload(type: "session_init_v2" | "project_init_v1") {
const segmentPrompts = getSessionInitPrompts();
setSeedPrompts(segmentPrompts);
const creationPayload = await buildCreationInitPayload({
modelId: creationModelId,
modeId: creationModeId,
aspectRatio: creationAspectRatio,
resolution: creationResolution,
durationSec: creationDurationSec,
referenceFile: referenceFileRef.current,
firstFrameFile: firstFrameFileRef.current,
lastFrameFile: lastFrameFileRef.current,
});
return {
type,
generation_mode: toGenerationMode(creationModeId),
preset_id: getInitialPresetId(),
preset_label: getInitialPresetLabel(),
curated_prompts: segmentPrompts,
initial_rollout_prompt: normalizeInitialPrompt(pendingInitialPromptRef.current),
initial_image: null,
single_clip_mode: false,
enhancement_enabled: sessionStore.get().enhancementEnabled,
auto_extension_enabled: sessionStore.get().autoExtensionEnabled,
loop_generation_enabled: sessionStore.get().loopGenerationEnabled,
...creationPayload,
};
}
function sendSessionInitMessage() {
async function sendSessionInitMessage() {
const ws = wsRef.current;
if (!ws) return;
ws.send(JSON.stringify(buildProjectInitPayload("session_init_v2")));
ws.send(JSON.stringify(await buildProjectInitPayload("session_init_v2")));
}
function sendProjectInitMessage() {
async function sendProjectInitMessage() {
const ws = wsRef.current;
if (!ws || ws.readyState !== WebSocket.OPEN) return;
ws.send(JSON.stringify(buildProjectInitPayload("project_init_v1")));
ws.send(JSON.stringify(await buildProjectInitPayload("project_init_v1")));
}
function sendEndProjectKeepSession() {
@@ -1822,6 +1898,9 @@ export default function Page() {
return;
}
const normalizedEvent = normalizeSocketMessage(decoded.data);
if (decoded.data?.type === "gpu_assigned" || decoded.data?.type === "ltx2_stream_start") {
applyEchoedCreationConfig(decoded.data);
}
await applyNormalizedSocketEvent(normalizedEvent, {
sessionStore,
promptWindowStore,
@@ -1864,7 +1943,12 @@ export default function Page() {
onOpen: () => {
opened = true;
sessionStore.patch({ connected: true, connecting: false });
sendSessionInitMessage();
void sendSessionInitMessage().catch((error) => {
console.error("Failed to send session init payload:", error);
recoverFailedSessionStart(
error instanceof Error ? error.message : "Failed to prepare session settings.",
);
});
},
onMessage: (event: MessageEvent) => {
wsMessageQueueRef.current = wsMessageQueueRef.current
@@ -1937,10 +2021,29 @@ export default function Page() {
}
}
function syncSessionCreationConfigFromLobby() {
setSessionCreationConfig({
modelId: creationModelId,
modeId: creationModeId,
aspectRatio: creationAspectRatio,
resolution: creationResolution,
durationSec: creationDurationSec,
});
}
function applyEchoedCreationConfig(data: unknown) {
const echoed = parseEchoedCreationConfig(data);
if (!echoed) {
return;
}
setSessionCreationConfig(echoed);
}
function beginProjectLocally({ force = false } = {}) {
if (!force && !canStartSession) return;
if (sessionStore.get().sessionStarted || sessionStore.get().projectResetPending) return false;
setTimeoutModalOpen(false);
syncSessionCreationConfigFromLobby();
// Unmute during the user gesture so iOS Safari permits audio playback.
setVideoMuted(false);
if (viewingProject) closeViewingProject();
@@ -1999,13 +2102,34 @@ export default function Page() {
}
async function joinSession({ force = false } = {}) {
const validationError = validateLobbyCreationSelection({
capabilities: activeModelCapabilities,
modelId: creationModelId,
modeId: creationModeId,
aspectRatio: creationAspectRatio,
resolution: creationResolution,
durationSec: creationDurationSec,
referenceFile: referenceFileRef.current,
firstFrameFile: firstFrameFileRef.current,
lastFrameFile: lastFrameFileRef.current,
});
if (validationError) {
showPreSessionNotice(validationError);
return;
}
if (
wsRef.current
&& wsRef.current.readyState === WebSocket.OPEN
&& sessionStore.get().connected
) {
if (!beginProjectLocally({ force })) return;
sendProjectInitMessage();
try {
await sendProjectInitMessage();
} catch (error) {
console.error("Failed to send project init payload:", error);
showPreSessionNotice(error instanceof Error ? error.message : "Failed to prepare session settings.");
}
return;
}
showPreSessionNotice("");
@@ -2027,7 +2151,12 @@ export default function Page() {
&& wsRef.current.readyState === WebSocket.OPEN
&& sessionStore.get().connected
) {
sendProjectInitMessage();
try {
await sendProjectInitMessage();
} catch (error) {
console.error("Failed to send project init payload:", error);
showPreSessionNotice(error instanceof Error ? error.message : "Failed to prepare session settings.");
}
return;
}
connectWebSocket();
@@ -2641,7 +2770,7 @@ export default function Page() {
/>
<Header timeLeft={headerTimeLeft} formatTime={formatTime} onToggleSidebar={() => setSidebarOpen((prev) => !prev)} />
<div className="relative flex flex-1 min-h-0 flex-col justify-center px-4 pb-2 sm:px-6 sm:pb-12">
<div className={cn("relative flex flex-1 min-h-0 flex-col", showActiveProject || isViewingMode ? "justify-center px-4 pb-2 sm:px-6 sm:pb-12" : "overflow-hidden")}>
{isViewingMode && (
<>
{viewingSelectedClip && (
@@ -2758,44 +2887,79 @@ export default function Page() {
/>
</section>
<AnimatePresence>
{!showActiveProject && (
<motion.div
key="hero-tagline"
initial={{ opacity: 0 }}
animate={{ opacity: 1 }}
exit={{ opacity: 0, transition: { duration: 0.2, ease: "easeIn" } }}
transition={{ duration: 0.5, ease: "easeOut" }}
className="pointer-events-none absolute inset-x-0 top-0 bottom-1/2 z-10 flex items-center justify-center px-4"
>
<HeroTagline />
</motion.div>
)}
</AnimatePresence>
<motion.div layout="position" className="mx-auto w-full max-w-2xl shrink-0" transition={{ type: "spring", stiffness: 200, damping: 25 }}>
<ChatBar
sessionStarted={sessionStarted as boolean}
rewritingSeedPrompts={rewritingSeedPrompts as boolean}
{!showActiveProject ? (
<CreationStudio
value={livePromptDraft as string}
disabled={projectResetPending as boolean}
isGenerating={loadingAnimation as boolean}
storyPresets={storyPresets as any[]}
continuationDraft={livePromptDraft as string}
canJoinSession={canStartSession}
canSubmitContinuation={canSubmitContinuation}
sessionExpired={sessionExpired as boolean}
sessionNotice={sessionNotice as string}
projectResetPending={projectResetPending as boolean}
canSubmit={canStartSession}
modelId={creationModelId}
modeId={creationModeId}
aspectRatio={creationAspectRatio}
resolution={creationResolution}
durationSec={creationDurationSec}
referencePreviewUrl={referencePreviewUrl}
firstFramePreviewUrl={firstFramePreviewUrl}
lastFramePreviewUrl={lastFramePreviewUrl}
mentionOptions={mentionOptions}
storyPresets={lobbyStoryPresets}
capabilities={activeModelCapabilities}
onValueChange={(value) => sessionStore.patch({ livePromptDraft: value })}
onSubmit={() => void joinSession()}
onKeyDown={handleLivePromptKeydown}
onModelChange={handleCreationModelChange}
onModeChange={setCreationModeId}
onAspectRatioChange={setCreationAspectRatio}
onResolutionChange={setCreationResolution}
onDurationChange={setCreationDurationSec}
onReferenceSelect={handleReferenceSelect}
onFirstFrameSelect={handleFirstFrameSelect}
onLastFrameSelect={handleLastFrameSelect}
onPresetGenerate={handlePresetGenerate}
onContinuationInput={handleLivePromptInput}
onContinuationKeydown={handleLivePromptKeydown}
onGenerate={joinSession}
onSubmitContinuation={submitLivePrompt}
onLeave={leaveSession}
onStartNewProject={handleStartNewProject}
onSpeechTranscript={handleLivePromptSpeechTranscript}
onSpeechInterimChange={handleLivePromptSpeechInterim}
onOpenProjects={() => setSidebarOpen(true)}
/>
</motion.div>
) : (
<motion.div layout="position" className="mx-auto w-full max-w-2xl shrink-0" transition={{ type: "spring", stiffness: 200, damping: 25 }}>
<ChatBar
sessionStarted={sessionStarted as boolean}
rewritingSeedPrompts={rewritingSeedPrompts as boolean}
isGenerating={loadingAnimation as boolean}
storyPresets={storyPresets as any[]}
continuationDraft={livePromptDraft as string}
canJoinSession={canStartSession}
canSubmitContinuation={canSubmitContinuation}
sessionExpired={sessionExpired as boolean}
sessionNotice={sessionNotice as string}
projectResetPending={projectResetPending as boolean}
sessionCreationConfig={sessionCreationConfig}
configPillsReadOnly
onSessionModelChange={(modelId) => setSessionCreationConfig((current) => ({ ...current, modelId }))}
onSessionModeChange={(modeId) => setSessionCreationConfig((current) => ({ ...current, modeId }))}
onSessionAspectRatioChange={(aspectRatio) => setSessionCreationConfig((current) => ({ ...current, aspectRatio }))}
onSessionResolutionChange={(resolution) => setSessionCreationConfig((current) => ({ ...current, resolution }))}
onSessionDurationChange={(durationSec) => setSessionCreationConfig((current) => ({ ...current, durationSec }))}
onPresetGenerate={handlePresetGenerate}
onContinuationInput={handleLivePromptInput}
onContinuationKeydown={handleLivePromptKeydown}
onGenerate={joinSession}
onSubmitContinuation={submitLivePrompt}
onLeave={leaveSession}
onStartNewProject={handleStartNewProject}
onSpeechTranscript={handleLivePromptSpeechTranscript}
onSpeechInterimChange={handleLivePromptSpeechInterim}
/>
</motion.div>
)}
{sessionNotice && !showActiveProject && (
<div className="mx-auto mt-2 w-full max-w-3xl px-4">
<div className="rounded-xl border border-rose-500/20 bg-rose-500/10 px-4 py-2.5 text-center text-xs text-rose-700 dark:text-rose-300">
{sessionNotice}
</div>
</div>
)}
</div>
</div>
</main>
+62 -197
View File
@@ -2,10 +2,13 @@
import React, { useRef, useState, useCallback, useEffect } from "react";
import Image from "next/image";
import { Film, ArrowUp, X, Loader2, ArrowLeft } from "lucide-react";
import { ArrowUp, X, Loader2, ArrowLeft } from "lucide-react";
import { Button } from "@/components/ui/button";
import LeaveSessionModal, { shouldShowLeaveWarning } from "@/components/LeaveSessionModal";
import PresetQuickLaunchRail from "@/components/creation/PresetQuickLaunchRail";
import SessionCreationConfigPills, { type SessionCreationConfig } from "@/components/creation/SessionCreationConfigPills";
import SpeechToTextButton from "@/components/SpeechToTextButton";
import type { AspectRatioId, CreationModeId, CreationModelId, ResolutionId } from "@/lib/creationConfig";
import { cn } from "@/lib/utils";
const PROMPT_MAX_LENGTH = 500;
@@ -32,6 +35,13 @@ interface Props {
onBackFromViewing?: () => void;
onSpeechTranscript?: (text: string) => void;
onSpeechInterimChange?: (text: string) => void;
sessionCreationConfig?: SessionCreationConfig | null;
configPillsReadOnly?: boolean;
onSessionModelChange?: (modelId: CreationModelId) => void;
onSessionModeChange?: (modeId: CreationModeId) => void;
onSessionAspectRatioChange?: (aspectRatio: AspectRatioId) => void;
onSessionResolutionChange?: (resolution: ResolutionId) => void;
onSessionDurationChange?: (durationSec: number) => void;
}
export default function ChatBar({
@@ -56,6 +66,13 @@ export default function ChatBar({
onBackFromViewing = () => {},
onSpeechTranscript,
onSpeechInterimChange,
sessionCreationConfig = null,
configPillsReadOnly = false,
onSessionModelChange,
onSessionModeChange,
onSessionAspectRatioChange,
onSessionResolutionChange,
onSessionDurationChange,
}: Props) {
const [sttBusy, setSttBusy] = useState(false);
const [leaveModalOpen, setLeaveModalOpen] = useState(false);
@@ -66,134 +83,11 @@ export default function ChatBar({
: isBusy
? "Generating video\u2026"
: !sessionStarted
? "What video are you imagining?"
? "Describe your video"
: "What do you want to edit?";
const actionLabel = !sessionStarted ? "Generate" : "Rewrite rollout";
const inputRef = useRef<HTMLTextAreaElement>(null);
const scrollRef = useRef<HTMLDivElement>(null);
const [canScrollLeft, setCanScrollLeft] = useState(false);
const [canScrollRight, setCanScrollRight] = useState(false);
const [presetRailDragging, setPresetRailDragging] = useState(false);
const presetDragStateRef = useRef({
pointerId: null as number | null,
startX: 0,
startScrollLeft: 0,
moved: false,
});
const suppressPresetClickRef = useRef(false);
const updateScrollState = useCallback(() => {
const el = scrollRef.current;
if (!el) return;
setCanScrollLeft(el.scrollLeft > 2);
setCanScrollRight(el.scrollLeft + el.clientWidth < el.scrollWidth - 2);
}, []);
const handlePresetWheel = useCallback(
(event: React.WheelEvent<HTMLDivElement>) => {
const el = scrollRef.current;
if (!el) return;
if (el.scrollWidth <= el.clientWidth + 1) return;
const dominantDelta = Math.abs(event.deltaX) > Math.abs(event.deltaY)
? event.deltaX
: event.deltaY;
if (!dominantDelta) return;
const maxScrollLeft = Math.max(el.scrollWidth - el.clientWidth, 0);
const nextScrollLeft = Math.min(
Math.max(el.scrollLeft + dominantDelta, 0),
maxScrollLeft,
);
if (nextScrollLeft === el.scrollLeft) return;
event.preventDefault();
el.scrollLeft = nextScrollLeft;
updateScrollState();
},
[updateScrollState],
);
const finishPresetDrag = useCallback(() => {
presetDragStateRef.current = {
pointerId: null,
startX: 0,
startScrollLeft: 0,
moved: false,
};
setPresetRailDragging(false);
}, []);
const handlePresetPointerDown = useCallback(
(event: React.PointerEvent<HTMLDivElement>) => {
const el = scrollRef.current;
if (!el) return;
if (event.pointerType !== "mouse" || event.button !== 0) return;
if (el.scrollWidth <= el.clientWidth + 1) return;
suppressPresetClickRef.current = false;
presetDragStateRef.current = {
pointerId: event.pointerId,
startX: event.clientX,
startScrollLeft: el.scrollLeft,
moved: false,
};
},
[],
);
const handlePresetPointerMove = useCallback(
(event: React.PointerEvent<HTMLDivElement>) => {
const el = scrollRef.current;
const dragState = presetDragStateRef.current;
if (!el || dragState.pointerId !== event.pointerId) return;
const deltaX = event.clientX - dragState.startX;
if (!dragState.moved && Math.abs(deltaX) > 4) {
dragState.moved = true;
suppressPresetClickRef.current = true;
setPresetRailDragging(true);
el.setPointerCapture?.(event.pointerId);
}
if (!dragState.moved) return;
event.preventDefault();
const maxScrollLeft = Math.max(el.scrollWidth - el.clientWidth, 0);
el.scrollLeft = Math.min(
Math.max(dragState.startScrollLeft - deltaX, 0),
maxScrollLeft,
);
updateScrollState();
},
[updateScrollState],
);
const handlePresetPointerUp = useCallback(
(event: React.PointerEvent<HTMLDivElement>) => {
const el = scrollRef.current;
if (!el || presetDragStateRef.current.pointerId !== event.pointerId) return;
if (el.hasPointerCapture?.(event.pointerId)) {
el.releasePointerCapture(event.pointerId);
}
finishPresetDrag();
},
[finishPresetDrag],
);
const handlePresetClickCapture = useCallback(
(event: React.MouseEvent<HTMLDivElement>) => {
if (!suppressPresetClickRef.current) return;
suppressPresetClickRef.current = false;
event.preventDefault();
event.stopPropagation();
},
[],
);
useEffect(() => {
updateScrollState();
}, [storyPresets, updateScrollState]);
useEffect(() => {
if (!isBusy && !sttBusy && !window.matchMedia("(pointer: coarse)").matches) {
@@ -239,7 +133,7 @@ export default function ChatBar({
<div className="flex flex-col items-center gap-3 rounded-2xl border border-border bg-card/80 px-6 py-4 text-center shadow-md backdrop-blur-sm">
<div className="flex flex-col gap-1">
<p className="text-sm font-semibold text-foreground">View-only project</p>
<p className="max-w-md text-xs text-muted-foreground">Project sessions are currently limited to 5 minutes. Start a new project to create more videos.</p>
<p className="max-w-md text-xs text-muted-foreground">Sessions are limited to 5 minutes. Start a new project to keep creating.</p>
</div>
<div className="mt-1 flex items-center gap-2">
<Button onClick={onBackFromViewing} variant="outline" size="sm" className="gap-1.5 rounded-full px-4">
@@ -261,7 +155,7 @@ export default function ChatBar({
<div className="flex flex-col items-center gap-3 rounded-2xl border border-border bg-card/80 px-8 py-5 text-center shadow-md backdrop-blur-sm">
<div className="flex flex-col gap-1">
<p className="text-sm font-semibold text-foreground">Session ended</p>
<p className="max-w-xs text-xs text-muted-foreground">Each project currently has a 5-minute session. Start a new project to continue creating videos.</p>
<p className="max-w-xs text-xs text-muted-foreground">Sessions are limited to 5 minutes. Start a new project to continue.</p>
</div>
<div className="mt-1 flex items-center gap-2">
<Button onClick={onStartNewProject} size="sm" className="rounded-full px-5">
@@ -280,51 +174,8 @@ export default function ChatBar({
return (
<section className="mx-auto flex w-full max-w-2xl shrink-0 flex-col gap-4">
{storyPresets.length > 0 && !sessionStarted && (
<div className={cn("relative transition-opacity duration-200", isGenerating && "pointer-events-none opacity-40")}>
<div
ref={scrollRef}
onScroll={updateScrollState}
onWheel={handlePresetWheel}
onPointerDown={handlePresetPointerDown}
onPointerMove={handlePresetPointerMove}
onPointerUp={handlePresetPointerUp}
onPointerCancel={handlePresetPointerUp}
onLostPointerCapture={finishPresetDrag}
onClickCapture={handlePresetClickCapture}
className={cn(
"scrollbar-hidden flex gap-3 overflow-x-auto px-1 select-none",
presetRailDragging ? "cursor-grabbing" : "cursor-grab",
)}
>
{storyPresets.map((preset) => (
<button
key={preset.id}
type="button"
disabled={isGenerating}
onClick={() => onPresetGenerate(preset.id)}
className="flex flex-col sm:flex-row items-start gap-1.5 shrink-0 rounded-xl border p-2.5 text-left backdrop-blur-sm transition-colors max-w-42 sm:max-w-[215px] border-input bg-card/80 text-muted-foreground hover:bg-slate-200/60 hover:border-slate-400 hover:text-slate-700 dark:bg-slate-800/80 dark:text-slate-300 dark:hover:bg-slate-700/50 dark:hover:border-slate-500 dark:hover:text-slate-200"
>
<Film className="mt-0.5 size-4 shrink-0 opacity-60" />
<span className="flex flex-col gap-1 min-w-0">
<span className="text-[14px] font-medium line-clamp-1">{preset.label}</span>
{preset.description && <span className="text-xs leading-tight opacity-70 line-clamp-3 sm:line-clamp-2">{preset.description}</span>}
</span>
</button>
))}
</div>
<div
className={cn("pointer-events-none absolute inset-y-0 left-0 w-8 bg-background transition-opacity duration-150", canScrollLeft ? "opacity-100" : "opacity-0")}
style={{ maskImage: "linear-gradient(to right, black, transparent)", WebkitMaskImage: "linear-gradient(to right, black, transparent)" }}
aria-hidden="true"
/>
<div
className={cn("pointer-events-none absolute inset-y-0 right-0 w-8 bg-background transition-opacity duration-150", canScrollRight ? "opacity-100" : "opacity-0")}
style={{ maskImage: "linear-gradient(to left, black, transparent)", WebkitMaskImage: "linear-gradient(to left, black, transparent)" }}
aria-hidden="true"
/>
</div>
{!sessionStarted && (
<PresetQuickLaunchRail storyPresets={storyPresets} disabled={isGenerating} onPresetGenerate={onPresetGenerate} />
)}
{sessionNotice && (
@@ -342,17 +193,30 @@ export default function ChatBar({
{projectResetPending && sessionStarted && (
<div className="rounded-xl border border-sky-500/20 bg-sky-500/10 px-4 py-2.5 text-center text-xs text-sky-700 dark:text-sky-300">
Starting a new project after the current shot finishes. Your GPU session stays active.
Starting a new project when this shot finishes. GPU session stays open.
</div>
)}
<div
className={cn(
"flex min-w-0 items-center gap-1.5 rounded-4xl border py-2.5 pl-5 pr-2.5 shadow-md backdrop-blur-sm transition-all duration-200",
"flex min-w-0 flex-col gap-2 rounded-4xl border py-2.5 pl-5 pr-2.5 shadow-md backdrop-blur-sm transition-all duration-200",
isBusy ? "border-input/60 bg-card/40" : "border-input bg-card/65",
)}
>
<textarea
{sessionStarted && sessionCreationConfig && (
<SessionCreationConfigPills
{...sessionCreationConfig}
disabled={isBusy}
readOnly={configPillsReadOnly}
onModelChange={onSessionModelChange}
onModeChange={onSessionModeChange}
onAspectRatioChange={onSessionAspectRatioChange}
onResolutionChange={onSessionResolutionChange}
onDurationChange={onSessionDurationChange}
/>
)}
<div className="flex min-w-0 items-center gap-1.5">
<textarea
ref={inputRef}
id="continuation-prompt"
aria-label="Continuation prompt"
@@ -367,36 +231,37 @@ export default function ChatBar({
"min-w-0 flex-1 resize-none bg-transparent text-foreground outline-none placeholder:text-muted-foreground transition-opacity duration-200 scrollbar-thin leading-snug",
(isBusy || sttBusy) && "cursor-not-allowed opacity-50",
)}
/>
{onSpeechTranscript && <SpeechToTextButton disabled={isBusy} onTranscript={onSpeechTranscript} onInterimChange={onSpeechInterimChange} onBusyChange={setSttBusy} />}
{!sessionStarted ? (
<Button
aria-label={actionLabel}
title={actionLabel}
onClick={onGenerate}
disabled={!canJoinSession || isGenerating || !continuationDraft.trim()}
size="icon-sm"
className="shrink-0 rounded-full"
>
{showSpinner ? <Loader2 className="size-5 animate-spin" /> : <ArrowUp className="size-5" />}
</Button>
) : (
<>
/>
{onSpeechTranscript && <SpeechToTextButton disabled={isBusy} onTranscript={onSpeechTranscript} onInterimChange={onSpeechInterimChange} onBusyChange={setSttBusy} />}
{!sessionStarted ? (
<Button
aria-label={actionLabel}
title={actionLabel}
onClick={onSubmitContinuation}
disabled={!canSubmitContinuation || showSpinner || projectResetPending || !continuationDraft.trim()}
onClick={onGenerate}
disabled={!canJoinSession || isGenerating || !continuationDraft.trim()}
size="icon-sm"
className="shrink-0 rounded-full"
>
{showSpinner ? <Loader2 className="size-5 animate-spin" /> : <ArrowUp className="size-5" />}
</Button>
<Button variant="outline" aria-label="Leave" title="Leave" onClick={() => { if (shouldShowLeaveWarning()) setLeaveModalOpen(true); else onLeave(); }} disabled={isGenerating || projectResetPending} size="icon-sm" className="shrink-0 rounded-full">
<X className="size-5" />
</Button>
</>
)}
) : (
<>
<Button
aria-label={actionLabel}
title={actionLabel}
onClick={onSubmitContinuation}
disabled={!canSubmitContinuation || showSpinner || projectResetPending || !continuationDraft.trim()}
size="icon-sm"
className="shrink-0 rounded-full"
>
{showSpinner ? <Loader2 className="size-5 animate-spin" /> : <ArrowUp className="size-5" />}
</Button>
<Button variant="outline" aria-label="Leave" title="Leave" onClick={() => { if (shouldShowLeaveWarning()) setLeaveModalOpen(true); else onLeave(); }} disabled={isGenerating || projectResetPending} size="icon-sm" className="shrink-0 rounded-full">
<X className="size-5" />
</Button>
</>
)}
</div>
</div>
<p className="px-2 text-center text-[11px] text-muted-foreground">
LLM powered by{" "}
@@ -0,0 +1,95 @@
"use client";
import { Fragment, useEffect, useRef } from "react";
const HERO_WAVE_LIGHT = ["#2A4A98", "#4878E5", "#6FA0F2", "#B0BCC8", "#E8D99E", "#D8C844", "#C2A620"];
const HERO_WAVE_DARK = ["#143468", "#1E58B8", "#3892F0", "#80B8E8", "#B8D0EA", "#E2D498", "#DABB50"];
const HERO_TEXT = "Direct scenes in seconds";
export default function HeroTagline() {
const ref = useRef<HTMLHeadingElement>(null);
useEffect(() => {
const el = ref.current;
if (!el) return;
let rafId = 0;
function play() {
const chars = el!.querySelectorAll<HTMLSpanElement>("[data-char]");
if (!chars.length) return;
cancelAnimationFrame(rafId);
const isDark = document.documentElement.classList.contains("dark");
const colors = isDark ? HERO_WAVE_DARK : HERO_WAVE_LIGHT;
const waveLen = 10;
const total = chars.length + waveLen;
const duration = 1200;
const maxBlur = 3.5;
const start = performance.now();
function tick() {
const t = Math.min((performance.now() - start) / duration, 1);
const pos = t * total;
chars.forEach((ch, i) => {
const rel = pos - i;
if (rel >= 0 && rel < waveLen) {
const norm = rel / waveLen;
const ci = Math.floor(norm * colors.length);
ch.style.color = colors[Math.min(colors.length - 1, ci)];
let blur = 0;
if (norm < 0.25) {
blur = maxBlur * (1 - norm / 0.25);
} else if (norm > 0.75) {
blur = maxBlur * ((norm - 0.75) / 0.25);
}
ch.style.filter = blur > 0.1 ? `blur(${blur.toFixed(1)}px)` : "";
} else {
ch.style.color = "";
ch.style.filter = "";
}
});
if (t < 1) {
rafId = requestAnimationFrame(tick);
} else {
chars.forEach((ch) => {
ch.style.color = "";
ch.style.filter = "";
});
}
}
rafId = requestAnimationFrame(tick);
}
const initialDelay = setTimeout(play, 400);
const interval = setInterval(play, 5000);
return () => {
clearTimeout(initialDelay);
clearInterval(interval);
cancelAnimationFrame(rafId);
};
}, []);
return (
<h1 ref={ref} className="text-balance text-center text-3xl font-medium text-[#343537] dark:text-[#FAFAFB] sm:text-4xl">
{HERO_TEXT.split(" ").map((word, wi) => (
<Fragment key={wi}>
{wi > 0 && (
<span data-char className="transition-[color,filter] duration-150">
{" "}
</span>
)}
<span className="inline-flex">
{word.split("").map((char, ci) => (
<span key={ci} data-char className="inline-block transition-[color,filter] duration-150">
{char}
</span>
))}
</span>
</Fragment>
))}
</h1>
);
}
@@ -0,0 +1,66 @@
"use client";
import React from "react";
import { FolderOpen, Home, Sparkles } from "lucide-react";
import { cn } from "@/lib/utils";
export type AppNavSection = "explore" | "create" | "assets";
interface AppNavRailProps {
activeSection?: AppNavSection;
onSectionChange?: (section: AppNavSection) => void;
onOpenProjects?: () => void;
className?: string;
}
const NAV_ITEMS: Array<{ id: AppNavSection; label: string; icon: typeof Home }> = [
{ id: "explore", label: "Explore", icon: Home },
{ id: "create", label: "Create", icon: Sparkles },
{ id: "assets", label: "Assets", icon: FolderOpen },
];
export default function AppNavRail({
activeSection = "create",
onSectionChange = () => {},
onOpenProjects,
className,
}: AppNavRailProps) {
return (
<aside
className={cn(
"hidden shrink-0 flex-col items-center gap-2 border-r border-border/40 bg-background/30 px-2.5 py-5 lg:flex",
className,
)}
aria-label="Primary navigation"
>
{NAV_ITEMS.map((item) => {
const Icon = item.icon;
const isActive = item.id === activeSection;
return (
<button
key={item.id}
type="button"
aria-label={item.label}
aria-current={isActive ? "page" : undefined}
onClick={() => {
if (item.id === "assets") {
onOpenProjects?.();
}
onSectionChange(item.id);
}}
className={cn(
"studio-control studio-control-press flex w-[4.5rem] min-h-11 flex-col items-center gap-1 rounded-xl px-2 py-2.5 text-[10px] font-medium tracking-wide",
isActive
? "bg-secondary/90 text-foreground shadow-sm ring-1 ring-border/60"
: "text-muted-foreground hover-capable:hover:bg-secondary/50 hover-capable:hover:text-foreground",
)}
>
<Icon className={cn("size-[18px]", isActive && "text-accent-blue")} />
{item.label}
</button>
);
})}
</aside>
);
}
@@ -0,0 +1,24 @@
"use client";
import React from "react";
import { cn } from "@/lib/utils";
export default function ConfigPill({
children,
className,
...props
}: React.ButtonHTMLAttributes<HTMLButtonElement>) {
return (
<button
type="button"
className={cn(
"studio-control studio-control-press studio-hover-surface inline-flex h-9 min-h-9 shrink-0 items-center gap-1 rounded-full border border-border/50 bg-background/80 px-2.5 text-[11px] font-medium text-foreground/90",
className,
)}
{...props}
>
{children}
</button>
);
}
@@ -0,0 +1,466 @@
"use client";
import React, { useMemo, useRef, useState } from "react";
import { ArrowUp, Box, ChevronDown, Clock, Monitor, Wand2 } from "lucide-react";
import ConfigPill from "@/components/creation/ConfigPill";
import HeroTagline from "@/components/HeroTagline";
import ReferenceUploadSlot from "@/components/creation/ReferenceUploadSlot";
import { Button } from "@/components/ui/button";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuLabel,
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { Slider } from "@/components/ui/slider";
import SpeechToTextButton from "@/components/SpeechToTextButton";
import {
ASPECT_RATIOS,
CREATION_MODELS,
CREATION_MODES,
RESOLUTIONS,
UNSUPPORTED_CREATION_MODES,
UNSUPPORTED_RESOLUTIONS,
modeRequiresReference,
modeUsesDualFrames,
type AspectRatioId,
type CreationModeId,
type CreationModelId,
type MentionOption,
type ResolutionId,
formatDurationLabel,
formatResolutionLabel,
} from "@/lib/creationConfig";
import {
DEFAULT_LOBBY_CAPABILITIES_BUNDLE,
isSupportedCreationMode,
isSupportedResolution,
resolveModelCapabilities,
unsupportedModeNotice,
type LobbyCreationCapabilities,
} from "@/lib/creationCapabilities";
import { cn } from "@/lib/utils";
const PROMPT_MAX_LENGTH = 500;
interface CreationComposerProps {
value: string;
disabled?: boolean;
isGenerating?: boolean;
canSubmit?: boolean;
modelId: CreationModelId;
modeId: CreationModeId;
aspectRatio: AspectRatioId;
resolution: ResolutionId;
durationSec: number;
referencePreviewUrl?: string | null;
firstFramePreviewUrl?: string | null;
lastFramePreviewUrl?: string | null;
mentionOptions?: MentionOption[];
onValueChange: (value: string) => void;
onSubmit: () => void;
onKeyDown?: (event: React.KeyboardEvent<HTMLTextAreaElement>) => void;
onModelChange: (modelId: CreationModelId) => void;
onModeChange: (modeId: CreationModeId) => void;
onAspectRatioChange: (aspectRatio: AspectRatioId) => void;
onResolutionChange: (resolution: ResolutionId) => void;
onDurationChange: (durationSec: number) => void;
onReferenceSelect?: (file: File | null) => void;
onFirstFrameSelect?: (file: File | null) => void;
onLastFrameSelect?: (file: File | null) => void;
onSpeechTranscript?: (text: string) => void;
onSpeechInterimChange?: (text: string) => void;
capabilities?: LobbyCreationCapabilities;
}
export default function CreationComposer({
value,
disabled = false,
isGenerating = false,
canSubmit = false,
modelId,
modeId,
aspectRatio,
resolution,
durationSec,
referencePreviewUrl = null,
firstFramePreviewUrl = null,
lastFramePreviewUrl = null,
mentionOptions = [],
onValueChange,
onSubmit,
onKeyDown,
onModelChange,
onModeChange,
onAspectRatioChange,
onResolutionChange,
onDurationChange,
onReferenceSelect,
onFirstFrameSelect,
onLastFrameSelect,
onSpeechTranscript,
onSpeechInterimChange,
capabilities = resolveModelCapabilities(DEFAULT_LOBBY_CAPABILITIES_BUNDLE, modelId),
}: CreationComposerProps) {
const inputRef = useRef<HTMLTextAreaElement>(null);
const [sttBusy, setSttBusy] = useState(false);
const [mentionQuery, setMentionQuery] = useState("");
const [mentionOpen, setMentionOpen] = useState(false);
const [mentionStart, setMentionStart] = useState<number | null>(null);
const availableModels = useMemo(
() => CREATION_MODELS.filter((model) => capabilities.model_ids.includes(model.id)),
[capabilities.model_ids],
);
const availableModes = useMemo(
() => CREATION_MODES.filter((mode) => isSupportedCreationMode(mode.id, capabilities)),
[capabilities],
);
const unavailableModes = useMemo(
() =>
UNSUPPORTED_CREATION_MODES.filter(
(mode) => unsupportedModeNotice(mode.id, capabilities) !== null,
),
[capabilities],
);
const availableAspectRatios = useMemo(
() => ASPECT_RATIOS.filter((ratio) => capabilities.aspect_ratios.includes(ratio)),
[capabilities.aspect_ratios],
);
const availableResolutions = useMemo(
() => RESOLUTIONS.filter((item) => isSupportedResolution(item, capabilities)),
[capabilities],
);
const unavailableResolutions = useMemo(
() => UNSUPPORTED_RESOLUTIONS.filter((item) => !isSupportedResolution(item, capabilities)),
[capabilities],
);
const durationMin = capabilities.duration_sec[0] ?? 5;
const durationMax = capabilities.duration_sec[capabilities.duration_sec.length - 1] ?? 15;
const selectedModel = availableModels.find((model) => model.id === modelId) ?? availableModels[0];
const selectedMode = availableModes.find((mode) => mode.id === modeId) ?? availableModes[0];
const usesDualFrames = modeUsesDualFrames(modeId);
const requiresReference = modeRequiresReference(modeId);
const referenceMissing = requiresReference && !referencePreviewUrl;
const submitDisabled = !canSubmit || disabled || isGenerating || !value.trim() || referenceMissing;
const filteredMentions = useMemo(() => {
const query = mentionQuery.trim().toLowerCase();
if (!query) return mentionOptions.slice(0, 6);
return mentionOptions
.filter((option) => option.label.toLowerCase().includes(query) || option.description?.toLowerCase().includes(query))
.slice(0, 6);
}, [mentionOptions, mentionQuery]);
function autoResize() {
const el = inputRef.current;
if (!el) return;
el.style.height = "auto";
const lineHeight = parseFloat(getComputedStyle(el).lineHeight) || 22;
const maxHeight = lineHeight * 4;
el.style.height = `${Math.min(el.scrollHeight, maxHeight)}px`;
el.style.overflowY = el.scrollHeight > maxHeight ? "auto" : "hidden";
}
function updateMentionState(nextValue: string, cursorPosition: number) {
const beforeCursor = nextValue.slice(0, cursorPosition);
const atIndex = beforeCursor.lastIndexOf("@");
if (atIndex === -1 || (atIndex > 0 && !/\s/.test(beforeCursor[atIndex - 1] ?? ""))) {
setMentionOpen(false);
setMentionStart(null);
setMentionQuery("");
return;
}
const query = beforeCursor.slice(atIndex + 1);
if (/\s/.test(query)) {
setMentionOpen(false);
setMentionStart(null);
setMentionQuery("");
return;
}
setMentionStart(atIndex);
setMentionQuery(query);
setMentionOpen(true);
}
function insertMention(option: MentionOption) {
if (mentionStart === null) return;
const before = value.slice(0, mentionStart);
const after = value.slice(inputRef.current?.selectionStart ?? value.length);
const mentionText = `@${option.label} `;
const nextValue = `${before}${mentionText}${after}`.slice(0, PROMPT_MAX_LENGTH);
onValueChange(nextValue);
setMentionOpen(false);
setMentionStart(null);
setMentionQuery("");
requestAnimationFrame(() => {
const el = inputRef.current;
if (!el) return;
const cursor = before.length + mentionText.length;
el.focus();
el.setSelectionRange(cursor, cursor);
autoResize();
});
}
function handleInputChange(event: React.ChangeEvent<HTMLTextAreaElement>) {
const nextValue = event.target.value.slice(0, PROMPT_MAX_LENGTH);
onValueChange(nextValue);
updateMentionState(nextValue, event.target.selectionStart ?? nextValue.length);
requestAnimationFrame(autoResize);
}
function handleKeyDown(event: React.KeyboardEvent<HTMLTextAreaElement>) {
if (mentionOpen && filteredMentions.length > 0) {
if (event.key === "Tab" || (event.key === "Enter" && !event.shiftKey)) {
event.preventDefault();
insertMention(filteredMentions[0]);
return;
}
if (event.key === "Escape") {
setMentionOpen(false);
return;
}
}
onKeyDown?.(event);
}
return (
<section className="mx-auto flex w-full max-w-3xl flex-col gap-5">
<HeroTagline />
<div className="rounded-[32px] border border-border/40 bg-secondary/95 p-4 shadow-[0_24px_80px_-32px_rgba(0,0,0,0.72)] backdrop-blur-xl sm:p-5">
<div className="flex gap-3.5">
{usesDualFrames ? (
<div className="flex shrink-0 gap-2">
<ReferenceUploadSlot
label="Asset"
sublabel="First"
previewUrl={firstFramePreviewUrl}
disabled={disabled}
onSelect={onFirstFrameSelect}
/>
<ReferenceUploadSlot
label="Asset"
sublabel="Last"
previewUrl={lastFramePreviewUrl}
disabled={disabled}
onSelect={onLastFrameSelect}
/>
</div>
) : (
<ReferenceUploadSlot
label="Reference"
previewUrl={referencePreviewUrl}
required={requiresReference}
optional={!requiresReference}
disabled={disabled}
onSelect={onReferenceSelect}
/>
)}
<div className="relative min-w-0 flex-1">
<textarea
ref={inputRef}
id="continuation-prompt"
aria-label="Continuation prompt"
value={value}
onChange={handleInputChange}
onKeyDown={handleKeyDown}
onClick={(event) => updateMentionState(value, event.currentTarget.selectionStart ?? value.length)}
placeholder="Describe your video or mention elements"
disabled={disabled || sttBusy}
rows={3}
className={cn(
"min-h-[92px] w-full resize-none bg-transparent px-0.5 text-base leading-6 text-foreground outline-none placeholder:text-muted-foreground/80 sm:text-sm",
(disabled || sttBusy) && "cursor-not-allowed opacity-50",
)}
/>
{mentionOpen && filteredMentions.length > 0 && (
<div className="absolute left-0 right-0 top-full z-20 mt-2 overflow-hidden rounded-2xl border border-border bg-popover/95 p-1 shadow-xl backdrop-blur-md">
<p className="px-2 py-1 text-[11px] font-medium uppercase tracking-wide text-muted-foreground">Mention</p>
{filteredMentions.map((option) => (
<button
key={option.id}
type="button"
onMouseDown={(event) => {
event.preventDefault();
insertMention(option);
}}
className="studio-control studio-hover-surface flex w-full items-start gap-2 rounded-xl px-2.5 py-2 text-left"
>
<span className="mt-0.5 rounded-md bg-accent px-1.5 py-0.5 text-[10px] font-semibold uppercase tracking-wide text-muted-foreground">
{option.kind}
</span>
<span className="min-w-0">
<span className="block truncate text-sm font-medium text-foreground">{option.label}</span>
{option.description && <span className="block truncate text-xs text-muted-foreground">{option.description}</span>}
</span>
</button>
))}
</div>
)}
</div>
</div>
<div className="mt-4 flex flex-wrap items-center gap-1.5 rounded-2xl bg-muted/35 p-1.5 ring-1 ring-border/25">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<ConfigPill disabled={disabled}>
<Box className="size-3.5" />
{selectedModel.label}
<ChevronDown className="size-3 opacity-60" />
</ConfigPill>
</DropdownMenuTrigger>
<DropdownMenuContent align="start" className="w-72">
<DropdownMenuLabel>Model</DropdownMenuLabel>
<DropdownMenuSeparator />
{availableModels.map((model) => (
<DropdownMenuItem key={model.id} onClick={() => onModelChange(model.id)} className="flex-col items-start gap-1 py-2.5">
<span className="flex items-center gap-2 text-sm font-medium">
{model.label}
{model.badge && <span className="rounded-full bg-accent-blue/15 px-1.5 py-0.5 text-[10px] text-accent-blue">{model.badge}</span>}
</span>
<span className="text-xs text-muted-foreground">{model.description}</span>
</DropdownMenuItem>
))}
</DropdownMenuContent>
</DropdownMenu>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<ConfigPill disabled={disabled}>
<Wand2 className="size-3.5" />
{selectedMode.label}
<ChevronDown className="size-3 opacity-60" />
</ConfigPill>
</DropdownMenuTrigger>
<DropdownMenuContent align="start" className="w-64">
<DropdownMenuLabel>Mode</DropdownMenuLabel>
<DropdownMenuSeparator />
{availableModes.map((mode) => (
<DropdownMenuItem key={mode.id} onClick={() => onModeChange(mode.id)} className="flex-col items-start gap-1 py-2.5">
<span className="text-sm font-medium">{mode.label}</span>
<span className="text-xs text-muted-foreground">{mode.description}</span>
</DropdownMenuItem>
))}
{unavailableModes.length > 0 && <DropdownMenuSeparator />}
{unavailableModes.map((mode) => (
<DropdownMenuItem key={mode.id} disabled className="flex-col items-start gap-1 py-2.5 opacity-60">
<span className="text-sm font-medium">{mode.label}</span>
<span className="text-xs text-muted-foreground">
{unsupportedModeNotice(mode.id, capabilities) ?? mode.description}
</span>
</DropdownMenuItem>
))}
</DropdownMenuContent>
</DropdownMenu>
<Popover>
<PopoverTrigger asChild>
<ConfigPill disabled={disabled}>
<Monitor className="size-3.5" />
{aspectRatio} {formatResolutionLabel(resolution)}
</ConfigPill>
</PopoverTrigger>
<PopoverContent align="start" className="w-80">
<p className="mb-3 text-xs font-medium text-muted-foreground">Aspect ratio</p>
<div className="grid grid-cols-3 gap-2">
{availableAspectRatios.map((ratio) => (
<button
key={ratio}
type="button"
onClick={() => onAspectRatioChange(ratio)}
className={cn(
"studio-control studio-control-press studio-hover-surface flex flex-col items-center gap-2 rounded-xl border px-2 py-3 text-xs",
aspectRatio === ratio ? "border-accent-blue bg-accent-blue/10 text-foreground" : "border-border",
)}
>
<span className={cn("rounded-sm border border-current/40 bg-muted/40", ratio === "9:16" && "h-7 w-4", ratio === "16:9" && "h-4 w-7", ratio === "1:1" && "size-5", ratio === "4:3" && "h-5 w-6", ratio === "3:4" && "h-6 w-5", ratio === "21:9" && "h-3 w-8")} />
{ratio}
</button>
))}
</div>
<p className="mb-2 mt-4 text-xs font-medium text-muted-foreground">Resolution</p>
<div className="flex flex-wrap gap-2">
{availableResolutions.map((item) => (
<button
key={item}
type="button"
onClick={() => onResolutionChange(item)}
className={cn(
"studio-control studio-control-press studio-hover-surface rounded-full border px-3 py-1.5 text-xs font-medium",
resolution === item ? "border-accent-blue bg-accent-blue/10 text-foreground" : "border-border",
)}
>
{formatResolutionLabel(item)}
</button>
))}
{unavailableResolutions.map((item) => (
<button
key={item}
type="button"
disabled
className="studio-control rounded-full border border-border px-3 py-1.5 text-xs font-medium text-muted-foreground opacity-50"
title="Not supported on FastLTX models yet"
>
{formatResolutionLabel(item)}
</button>
))}
</div>
</PopoverContent>
</Popover>
<Popover>
<PopoverTrigger asChild>
<ConfigPill disabled={disabled}>
<Clock className="size-3.5" />
{formatDurationLabel(durationSec)}
</ConfigPill>
</PopoverTrigger>
<PopoverContent align="start" className="w-72">
<p className="mb-3 text-xs font-medium text-muted-foreground">Total duration</p>
<Slider min={durationMin} max={durationMax} step={5} value={[durationSec]} onValueChange={(values) => onDurationChange(values[0] ?? durationMin)} />
<div className="mt-3 flex items-center justify-between text-[11px] text-muted-foreground">
<span>{formatDurationLabel(durationMin)}</span>
<span className="rounded-md border border-border px-2 py-1 text-xs font-medium text-foreground">{formatDurationLabel(durationSec)}</span>
<span>{formatDurationLabel(durationMax)}</span>
</div>
</PopoverContent>
</Popover>
<div className="ml-auto flex items-center gap-1.5">
{onSpeechTranscript && (
<SpeechToTextButton
disabled={disabled || isGenerating}
onTranscript={onSpeechTranscript}
onInterimChange={onSpeechInterimChange}
onBusyChange={setSttBusy}
/>
)}
<Button
aria-label="Generate"
onClick={onSubmit}
disabled={submitDisabled}
size="icon"
className="studio-control-press rounded-full bg-accent-blue text-white shadow-sm hover-capable:hover:bg-accent-blue/90 disabled:bg-muted disabled:text-muted-foreground"
>
<ArrowUp className="size-5" />
</Button>
</div>
</div>
{referenceMissing && value.trim() && (
<p className="mt-3 text-center text-xs leading-5 text-amber-700 dark:text-amber-400">
Upload a reference asset to use Omni reference mode.
</p>
)}
</div>
</section>
);
}
@@ -0,0 +1,73 @@
"use client";
import React from "react";
import AppNavRail, { type AppNavSection } from "@/components/creation/AppNavRail";
import CreationComposer from "@/components/creation/CreationComposer";
import PresetQuickLaunchRail, { type StoryPresetLike } from "@/components/creation/PresetQuickLaunchRail";
import {
type AspectRatioId,
type CreationModeId,
type CreationModelId,
type MentionOption,
type ResolutionId,
} from "@/lib/creationConfig";
import type { LobbyCreationCapabilities } from "@/lib/creationCapabilities";
interface CreationStudioProps {
value: string;
disabled?: boolean;
isGenerating?: boolean;
canSubmit?: boolean;
modelId: CreationModelId;
modeId: CreationModeId;
aspectRatio: AspectRatioId;
resolution: ResolutionId;
durationSec: number;
referencePreviewUrl?: string | null;
firstFramePreviewUrl?: string | null;
lastFramePreviewUrl?: string | null;
mentionOptions?: MentionOption[];
storyPresets?: StoryPresetLike[];
activeSection?: AppNavSection;
onValueChange: (value: string) => void;
onSubmit: () => void;
onKeyDown?: (event: React.KeyboardEvent<HTMLTextAreaElement>) => void;
onModelChange: (modelId: CreationModelId) => void;
onModeChange: (modeId: CreationModeId) => void;
onAspectRatioChange: (aspectRatio: AspectRatioId) => void;
onResolutionChange: (resolution: ResolutionId) => void;
onDurationChange: (durationSec: number) => void;
onReferenceSelect?: (file: File | null) => void;
onFirstFrameSelect?: (file: File | null) => void;
onLastFrameSelect?: (file: File | null) => void;
onPresetGenerate?: (presetId: string) => void;
onSpeechTranscript?: (text: string) => void;
onSpeechInterimChange?: (text: string) => void;
onOpenProjects?: () => void;
capabilities?: LobbyCreationCapabilities;
}
export default function CreationStudio({
activeSection = "create",
onOpenProjects,
storyPresets = [],
onPresetGenerate,
isGenerating = false,
capabilities,
...composerProps
}: CreationStudioProps) {
return (
<div className="flex min-h-0 flex-1">
<AppNavRail activeSection={activeSection} onOpenProjects={onOpenProjects} />
<div className="min-w-0 flex-1 overflow-y-auto">
<div className="mx-auto flex w-full max-w-5xl flex-col gap-5 px-4 py-7 sm:px-6 sm:py-8">
<CreationComposer {...composerProps} isGenerating={isGenerating} capabilities={capabilities} />
{storyPresets.length > 0 && onPresetGenerate && (
<PresetQuickLaunchRail storyPresets={storyPresets} disabled={isGenerating} onPresetGenerate={onPresetGenerate} />
)}
</div>
</div>
</div>
);
}
@@ -0,0 +1,240 @@
"use client";
import React, { useCallback, useEffect, useRef, useState } from "react";
import { ChevronLeft, ChevronRight } from "lucide-react";
import { cn } from "@/lib/utils";
export interface StoryPresetLike {
id: string;
label: string;
description?: string;
segmentCount?: number;
styleTag?: string;
}
interface PresetQuickLaunchRailProps {
storyPresets: StoryPresetLike[];
disabled?: boolean;
onPresetGenerate: (presetId: string) => void;
}
export default function PresetQuickLaunchRail({
storyPresets,
disabled = false,
onPresetGenerate,
}: PresetQuickLaunchRailProps) {
const scrollRef = useRef<HTMLDivElement>(null);
const [canScrollLeft, setCanScrollLeft] = useState(false);
const [canScrollRight, setCanScrollRight] = useState(false);
const [presetRailDragging, setPresetRailDragging] = useState(false);
const presetDragStateRef = useRef({
pointerId: null as number | null,
startX: 0,
startScrollLeft: 0,
moved: false,
});
const suppressPresetClickRef = useRef(false);
const updateScrollState = useCallback(() => {
const el = scrollRef.current;
if (!el) return;
setCanScrollLeft(el.scrollLeft > 2);
setCanScrollRight(el.scrollLeft + el.clientWidth < el.scrollWidth - 2);
}, []);
const scrollByAmount = useCallback(
(direction: "left" | "right") => {
const el = scrollRef.current;
if (!el) return;
const delta = direction === "left" ? -220 : 220;
el.scrollBy({ left: delta, behavior: "smooth" });
window.setTimeout(updateScrollState, 220);
},
[updateScrollState],
);
const handlePresetWheel = useCallback(
(event: React.WheelEvent<HTMLDivElement>) => {
const el = scrollRef.current;
if (!el) return;
if (el.scrollWidth <= el.clientWidth + 1) return;
const dominantDelta = Math.abs(event.deltaX) > Math.abs(event.deltaY) ? event.deltaX : event.deltaY;
if (!dominantDelta) return;
const maxScrollLeft = Math.max(el.scrollWidth - el.clientWidth, 0);
const nextScrollLeft = Math.min(Math.max(el.scrollLeft + dominantDelta, 0), maxScrollLeft);
if (nextScrollLeft === el.scrollLeft) return;
event.preventDefault();
el.scrollLeft = nextScrollLeft;
updateScrollState();
},
[updateScrollState],
);
const finishPresetDrag = useCallback(() => {
presetDragStateRef.current = {
pointerId: null,
startX: 0,
startScrollLeft: 0,
moved: false,
};
setPresetRailDragging(false);
}, []);
const handlePresetPointerDown = useCallback((event: React.PointerEvent<HTMLDivElement>) => {
const el = scrollRef.current;
if (!el) return;
if (event.pointerType !== "mouse" || event.button !== 0) return;
if (el.scrollWidth <= el.clientWidth + 1) return;
suppressPresetClickRef.current = false;
presetDragStateRef.current = {
pointerId: event.pointerId,
startX: event.clientX,
startScrollLeft: el.scrollLeft,
moved: false,
};
}, []);
const handlePresetPointerMove = useCallback(
(event: React.PointerEvent<HTMLDivElement>) => {
const el = scrollRef.current;
const dragState = presetDragStateRef.current;
if (!el || dragState.pointerId !== event.pointerId) return;
const deltaX = event.clientX - dragState.startX;
if (!dragState.moved && Math.abs(deltaX) > 4) {
dragState.moved = true;
suppressPresetClickRef.current = true;
setPresetRailDragging(true);
el.setPointerCapture?.(event.pointerId);
}
if (!dragState.moved) return;
event.preventDefault();
const maxScrollLeft = Math.max(el.scrollWidth - el.clientWidth, 0);
el.scrollLeft = Math.min(Math.max(dragState.startScrollLeft - deltaX, 0), maxScrollLeft);
updateScrollState();
},
[updateScrollState],
);
const handlePresetPointerUp = useCallback(
(event: React.PointerEvent<HTMLDivElement>) => {
const el = scrollRef.current;
if (!el || presetDragStateRef.current.pointerId !== event.pointerId) return;
if (el.hasPointerCapture?.(event.pointerId)) {
el.releasePointerCapture(event.pointerId);
}
finishPresetDrag();
},
[finishPresetDrag],
);
const handlePresetClickCapture = useCallback((event: React.MouseEvent<HTMLDivElement>) => {
if (!suppressPresetClickRef.current) return;
suppressPresetClickRef.current = false;
event.preventDefault();
event.stopPropagation();
}, []);
useEffect(() => {
updateScrollState();
}, [storyPresets, updateScrollState]);
useEffect(() => {
const el = scrollRef.current;
if (!el) return;
const observer = new ResizeObserver(() => updateScrollState());
observer.observe(el);
return () => observer.disconnect();
}, [updateScrollState]);
if (storyPresets.length === 0) return null;
const scrollMaskStyle =
canScrollLeft && canScrollRight
? {
maskImage: "linear-gradient(to right, transparent, black 20px, black calc(100% - 20px), transparent)",
WebkitMaskImage: "linear-gradient(to right, transparent, black 20px, black calc(100% - 20px), transparent)",
}
: canScrollLeft
? {
maskImage: "linear-gradient(to right, transparent, black 20px, black)",
WebkitMaskImage: "linear-gradient(to right, transparent, black 20px, black)",
}
: canScrollRight
? {
maskImage: "linear-gradient(to right, black, black calc(100% - 20px), transparent)",
WebkitMaskImage: "linear-gradient(to right, black, black calc(100% - 20px), transparent)",
}
: undefined;
return (
<div className={cn("mx-auto w-full max-w-3xl transition-opacity duration-200", disabled && "pointer-events-none opacity-40")}>
<div className="grid grid-cols-[auto_minmax(0,1fr)_auto] items-center gap-1 sm:gap-2">
<div className="flex w-8 shrink-0 justify-center">
{canScrollLeft ? (
<button
type="button"
aria-label="Scroll suggested prompts left"
onClick={() => scrollByAmount("left")}
className="studio-control studio-control-press inline-flex size-8 items-center justify-center rounded-full text-muted-foreground hover-capable:hover:bg-muted/60 hover-capable:hover:text-foreground"
>
<ChevronLeft className="size-4" />
</button>
) : null}
</div>
<div
ref={scrollRef}
onScroll={updateScrollState}
onWheel={handlePresetWheel}
onPointerDown={handlePresetPointerDown}
onPointerMove={handlePresetPointerMove}
onPointerUp={handlePresetPointerUp}
onPointerCancel={handlePresetPointerUp}
onLostPointerCapture={finishPresetDrag}
onClickCapture={handlePresetClickCapture}
style={scrollMaskStyle}
className={cn(
"scrollbar-hidden flex gap-2 overflow-x-auto overflow-y-visible py-0.5 select-none",
presetRailDragging ? "cursor-grabbing" : "cursor-grab",
)}
>
{storyPresets.map((preset) => (
<button
key={preset.id}
type="button"
disabled={disabled}
onClick={() => onPresetGenerate(preset.id)}
className="studio-control studio-control-press studio-hover-surface flex w-[12.5rem] shrink-0 flex-col gap-1 rounded-xl border border-border/50 bg-card/70 px-3 py-2.5 text-left"
>
<span className="line-clamp-1 text-sm font-medium text-foreground">{preset.label}</span>
{preset.description && (
<span className="text-pretty line-clamp-2 text-xs leading-5 text-muted-foreground">{preset.description}</span>
)}
</button>
))}
</div>
<div className="flex w-8 shrink-0 justify-center">
{canScrollRight ? (
<button
type="button"
aria-label="Scroll suggested prompts right"
onClick={() => scrollByAmount("right")}
className="studio-control studio-control-press inline-flex size-8 items-center justify-center rounded-full text-muted-foreground hover-capable:hover:bg-muted/60 hover-capable:hover:text-foreground"
>
<ChevronRight className="size-4" />
</button>
) : null}
</div>
</div>
</div>
);
}
@@ -0,0 +1,97 @@
"use client";
import React, { useRef, useState } from "react";
import { ImagePlus } from "lucide-react";
import { REFERENCE_ACCEPT, isReferenceMediaFile } from "@/lib/creationConfig";
import { cn } from "@/lib/utils";
interface ReferenceUploadSlotProps {
label: string;
sublabel?: string;
previewUrl?: string | null;
required?: boolean;
optional?: boolean;
disabled?: boolean;
onSelect?: (file: File | null) => void;
}
export default function ReferenceUploadSlot({
label,
sublabel,
previewUrl = null,
required = false,
optional = false,
disabled = false,
onSelect,
}: ReferenceUploadSlotProps) {
const fileInputRef = useRef<HTMLInputElement>(null);
const [dragActive, setDragActive] = useState(false);
function handleFile(file: File | null) {
if (!file || !isReferenceMediaFile(file)) return;
onSelect?.(file);
}
return (
<div className="flex flex-col gap-1">
<button
type="button"
aria-label={[label, sublabel].filter(Boolean).join(" ")}
onClick={() => fileInputRef.current?.click()}
disabled={disabled}
onDragEnter={(event) => {
event.preventDefault();
event.stopPropagation();
if (!disabled) setDragActive(true);
}}
onDragOver={(event) => {
event.preventDefault();
event.stopPropagation();
if (!disabled) setDragActive(true);
}}
onDragLeave={(event) => {
event.preventDefault();
event.stopPropagation();
setDragActive(false);
}}
onDrop={(event) => {
event.preventDefault();
event.stopPropagation();
setDragActive(false);
if (disabled) return;
handleFile(event.dataTransfer.files?.[0] ?? null);
}}
className={cn(
"studio-control studio-control-press studio-hover-surface relative flex size-[76px] shrink-0 flex-col items-center justify-center gap-1 overflow-hidden rounded-2xl border border-dashed bg-muted/50 px-1 text-center text-[11px] font-medium text-muted-foreground",
required && !previewUrl ? "border-amber-500/50" : "border-border/60",
dragActive && "border-accent-blue bg-accent-blue/10 ring-2 ring-accent-blue/30",
disabled && "pointer-events-none opacity-50",
)}
>
{previewUrl ? (
<img src={previewUrl} alt="" className="studio-media-outline absolute inset-0 size-full object-cover" />
) : (
<>
<ImagePlus className="size-4" />
<span>{label}</span>
{sublabel && <span className="text-[10px] font-normal opacity-70">{sublabel}</span>}
</>
)}
</button>
{(required || optional) && (
<span className="text-center text-[10px] text-muted-foreground">{required ? "Required" : "Optional"}</span>
)}
<input
ref={fileInputRef}
type="file"
accept={REFERENCE_ACCEPT}
className="hidden"
onChange={(event) => {
handleFile(event.target.files?.[0] ?? null);
event.target.value = "";
}}
/>
</div>
);
}
@@ -0,0 +1,213 @@
"use client";
import React from "react";
import { Box, ChevronDown, Clock, Monitor, Wand2 } from "lucide-react";
import ConfigPill from "@/components/creation/ConfigPill";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuLabel,
DropdownMenuSeparator,
DropdownMenuTrigger,
} from "@/components/ui/dropdown-menu";
import { Popover, PopoverContent, PopoverTrigger } from "@/components/ui/popover";
import { Slider } from "@/components/ui/slider";
import {
ASPECT_RATIOS,
CREATION_MODELS,
CREATION_MODES,
RESOLUTIONS,
type AspectRatioId,
type CreationModeId,
type CreationModelId,
type ResolutionId,
formatDurationLabel,
formatResolutionLabel,
} from "@/lib/creationConfig";
import { cn } from "@/lib/utils";
export interface SessionCreationConfig {
modelId: CreationModelId;
modeId: CreationModeId;
aspectRatio: AspectRatioId;
resolution: ResolutionId;
durationSec: number;
}
interface SessionCreationConfigPillsProps extends SessionCreationConfig {
disabled?: boolean;
readOnly?: boolean;
onModelChange?: (modelId: CreationModelId) => void;
onModeChange?: (modeId: CreationModeId) => void;
onAspectRatioChange?: (aspectRatio: AspectRatioId) => void;
onResolutionChange?: (resolution: ResolutionId) => void;
onDurationChange?: (durationSec: number) => void;
}
export default function SessionCreationConfigPills({
modelId,
modeId,
aspectRatio,
resolution,
durationSec,
disabled = false,
readOnly = false,
onModelChange,
onModeChange,
onAspectRatioChange,
onResolutionChange,
onDurationChange,
}: SessionCreationConfigPillsProps) {
const selectedModel = CREATION_MODELS.find((model) => model.id === modelId) ?? CREATION_MODELS[0];
const selectedMode = CREATION_MODES.find((mode) => mode.id === modeId) ?? CREATION_MODES[0];
const isInteractive = !readOnly && !disabled;
const pillClassName = cn(
"h-9 min-h-9 px-2 text-[11px]",
!isInteractive && "pointer-events-none opacity-70",
);
if (readOnly) {
return (
<div className="flex flex-wrap items-center gap-1.5">
<ConfigPill disabled className={pillClassName} aria-label="Model">
<Box className="size-3" />
{selectedModel.label}
</ConfigPill>
<ConfigPill disabled className={pillClassName} aria-label="Mode">
<Wand2 className="size-3" />
{selectedMode.label}
</ConfigPill>
<ConfigPill disabled className={pillClassName} aria-label="Aspect ratio and resolution">
<Monitor className="size-3" />
{aspectRatio} {formatResolutionLabel(resolution)}
</ConfigPill>
<ConfigPill disabled className={pillClassName} aria-label="Duration">
<Clock className="size-3" />
{formatDurationLabel(durationSec)}
</ConfigPill>
</div>
);
}
return (
<div className="flex flex-wrap items-center gap-1.5">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<ConfigPill disabled={disabled} className={pillClassName} aria-label="Model">
<Box className="size-3" />
{selectedModel.label}
<ChevronDown className="size-2.5 opacity-60" />
</ConfigPill>
</DropdownMenuTrigger>
<DropdownMenuContent align="start" className="w-72">
<DropdownMenuLabel>Model</DropdownMenuLabel>
<DropdownMenuSeparator />
{CREATION_MODELS.map((model) => (
<DropdownMenuItem key={model.id} onClick={() => onModelChange?.(model.id)} className="flex-col items-start gap-1 py-2.5">
<span className="flex items-center gap-2 text-sm font-medium">
{model.label}
{model.badge && <span className="rounded-full bg-accent-blue/15 px-1.5 py-0.5 text-[10px] text-accent-blue">{model.badge}</span>}
</span>
<span className="text-xs text-muted-foreground">{model.description}</span>
</DropdownMenuItem>
))}
</DropdownMenuContent>
</DropdownMenu>
<DropdownMenu>
<DropdownMenuTrigger asChild>
<ConfigPill disabled={disabled} className={pillClassName} aria-label="Mode">
<Wand2 className="size-3" />
{selectedMode.label}
<ChevronDown className="size-2.5 opacity-60" />
</ConfigPill>
</DropdownMenuTrigger>
<DropdownMenuContent align="start" className="w-64">
<DropdownMenuLabel>Mode</DropdownMenuLabel>
<DropdownMenuSeparator />
{CREATION_MODES.map((mode) => (
<DropdownMenuItem key={mode.id} onClick={() => onModeChange?.(mode.id)} className="flex-col items-start gap-1 py-2.5">
<span className="text-sm font-medium">{mode.label}</span>
<span className="text-xs text-muted-foreground">{mode.description}</span>
</DropdownMenuItem>
))}
</DropdownMenuContent>
</DropdownMenu>
<Popover>
<PopoverTrigger asChild>
<ConfigPill disabled={disabled} className={pillClassName} aria-label="Aspect ratio and resolution">
<Monitor className="size-3" />
{aspectRatio} {formatResolutionLabel(resolution)}
</ConfigPill>
</PopoverTrigger>
<PopoverContent align="start" className="w-80">
<p className="mb-3 text-xs font-medium text-muted-foreground">Aspect ratio</p>
<div className="grid grid-cols-3 gap-2">
{ASPECT_RATIOS.map((ratio) => (
<button
key={ratio}
type="button"
onClick={() => onAspectRatioChange?.(ratio)}
className={cn(
"studio-control studio-control-press studio-hover-surface flex flex-col items-center gap-2 rounded-xl border px-2 py-3 text-xs",
aspectRatio === ratio ? "border-accent-blue bg-accent-blue/10 text-foreground" : "border-border",
)}
>
<span
className={cn(
"rounded-sm border border-current/40 bg-muted/40",
ratio === "9:16" && "h-7 w-4",
ratio === "16:9" && "h-4 w-7",
ratio === "1:1" && "size-5",
ratio === "4:3" && "h-5 w-6",
ratio === "3:4" && "h-6 w-5",
ratio === "21:9" && "h-3 w-8",
)}
/>
{ratio}
</button>
))}
</div>
<p className="mb-2 mt-4 text-xs font-medium text-muted-foreground">Resolution</p>
<div className="flex flex-wrap gap-2">
{RESOLUTIONS.map((item) => (
<button
key={item}
type="button"
onClick={() => onResolutionChange?.(item)}
className={cn(
"studio-control studio-control-press studio-hover-surface rounded-full border px-3 py-1.5 text-xs font-medium",
resolution === item ? "border-accent-blue bg-accent-blue/10 text-foreground" : "border-border",
)}
>
{formatResolutionLabel(item)}
</button>
))}
</div>
</PopoverContent>
</Popover>
<Popover>
<PopoverTrigger asChild>
<ConfigPill disabled={disabled} className={pillClassName} aria-label="Duration">
<Clock className="size-3" />
{formatDurationLabel(durationSec)}
</ConfigPill>
</PopoverTrigger>
<PopoverContent align="start" className="w-72">
<p className="mb-3 text-xs font-medium text-muted-foreground">Total duration</p>
<Slider min={5} max={15} step={5} value={[durationSec]} onValueChange={(values) => onDurationChange?.(values[0] ?? 5)} />
<div className="mt-3 flex items-center justify-between text-[11px] text-muted-foreground">
<span>5s</span>
<span className="rounded-md border border-border px-2 py-1 text-xs font-medium text-foreground">{formatDurationLabel(durationSec)}</span>
<span>15s</span>
</div>
</PopoverContent>
</Popover>
</div>
);
}
@@ -0,0 +1,141 @@
"use client";
import * as React from "react";
import * as DropdownMenuPrimitive from "@radix-ui/react-dropdown-menu";
import { Check, ChevronRight } from "lucide-react";
import { cn } from "@/lib/utils";
const DropdownMenu = DropdownMenuPrimitive.Root;
const DropdownMenuTrigger = DropdownMenuPrimitive.Trigger;
const DropdownMenuGroup = DropdownMenuPrimitive.Group;
const DropdownMenuPortal = DropdownMenuPrimitive.Portal;
const DropdownMenuSub = DropdownMenuPrimitive.Sub;
const DropdownMenuRadioGroup = DropdownMenuPrimitive.RadioGroup;
const DropdownMenuSubTrigger = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.SubTrigger>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.SubTrigger> & { inset?: boolean }
>(({ className, inset, children, ...props }, ref) => (
<DropdownMenuPrimitive.SubTrigger
ref={ref}
className={cn(
"flex cursor-default select-none items-center rounded-xl px-2 py-1.5 text-sm outline-none data-[state=open]:bg-accent focus:bg-accent",
inset && "pl-8",
className,
)}
{...props}
>
{children}
<ChevronRight className="ml-auto size-4" />
</DropdownMenuPrimitive.SubTrigger>
));
DropdownMenuSubTrigger.displayName = DropdownMenuPrimitive.SubTrigger.displayName;
const DropdownMenuSubContent = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.SubContent>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.SubContent>
>(({ className, ...props }, ref) => (
<DropdownMenuPrimitive.SubContent
ref={ref}
className={cn(
"z-50 min-w-[8rem] overflow-hidden rounded-2xl border border-border bg-popover/95 p-1 text-popover-foreground shadow-xl backdrop-blur-md",
"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95",
"data-[side=bottom]:slide-in-from-top-2 data-[side=top]:slide-in-from-bottom-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2",
className,
)}
{...props}
/>
));
DropdownMenuSubContent.displayName = DropdownMenuPrimitive.SubContent.displayName;
const DropdownMenuContent = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.Content>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.Content>
>(({ className, sideOffset = 6, ...props }, ref) => (
<DropdownMenuPrimitive.Portal>
<DropdownMenuPrimitive.Content
ref={ref}
sideOffset={sideOffset}
className={cn(
"z-50 min-w-[12rem] overflow-hidden rounded-2xl border border-border bg-popover/95 p-1.5 text-popover-foreground shadow-xl backdrop-blur-md",
"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95",
"data-[side=bottom]:slide-in-from-top-2 data-[side=top]:slide-in-from-bottom-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2",
className,
)}
{...props}
/>
</DropdownMenuPrimitive.Portal>
));
DropdownMenuContent.displayName = DropdownMenuPrimitive.Content.displayName;
const DropdownMenuItem = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.Item>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.Item> & { inset?: boolean }
>(({ className, inset, ...props }, ref) => (
<DropdownMenuPrimitive.Item
ref={ref}
className={cn(
"relative flex cursor-default select-none items-center gap-2 rounded-xl px-2.5 py-2 text-sm outline-none transition-colors data-[disabled]:pointer-events-none data-[disabled]:opacity-50 focus:bg-accent focus:text-accent-foreground",
inset && "pl-8",
className,
)}
{...props}
/>
));
DropdownMenuItem.displayName = DropdownMenuPrimitive.Item.displayName;
const DropdownMenuCheckboxItem = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.CheckboxItem>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.CheckboxItem>
>(({ className, children, checked, ...props }, ref) => (
<DropdownMenuPrimitive.CheckboxItem
ref={ref}
className={cn(
"relative flex cursor-default select-none items-center rounded-xl py-2 pl-8 pr-2 text-sm outline-none transition-colors data-[disabled]:pointer-events-none data-[disabled]:opacity-50 focus:bg-accent focus:text-accent-foreground",
className,
)}
checked={checked}
{...props}
>
<span className="absolute left-2 flex size-3.5 items-center justify-center">
<DropdownMenuPrimitive.ItemIndicator>
<Check className="size-4 text-accent-blue" />
</DropdownMenuPrimitive.ItemIndicator>
</span>
{children}
</DropdownMenuPrimitive.CheckboxItem>
));
DropdownMenuCheckboxItem.displayName = DropdownMenuPrimitive.CheckboxItem.displayName;
const DropdownMenuLabel = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.Label>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.Label> & { inset?: boolean }
>(({ className, inset, ...props }, ref) => (
<DropdownMenuPrimitive.Label ref={ref} className={cn("px-2 py-1.5 text-xs font-semibold text-muted-foreground", inset && "pl-8", className)} {...props} />
));
DropdownMenuLabel.displayName = DropdownMenuPrimitive.Label.displayName;
const DropdownMenuSeparator = React.forwardRef<
React.ElementRef<typeof DropdownMenuPrimitive.Separator>,
React.ComponentPropsWithoutRef<typeof DropdownMenuPrimitive.Separator>
>(({ className, ...props }, ref) => (
<DropdownMenuPrimitive.Separator ref={ref} className={cn("-mx-1 my-1 h-px bg-border", className)} {...props} />
));
DropdownMenuSeparator.displayName = DropdownMenuPrimitive.Separator.displayName;
export {
DropdownMenu,
DropdownMenuTrigger,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuCheckboxItem,
DropdownMenuLabel,
DropdownMenuSeparator,
DropdownMenuGroup,
DropdownMenuPortal,
DropdownMenuSub,
DropdownMenuSubContent,
DropdownMenuSubTrigger,
DropdownMenuRadioGroup,
};
@@ -0,0 +1,33 @@
"use client";
import * as React from "react";
import * as PopoverPrimitive from "@radix-ui/react-popover";
import { cn } from "@/lib/utils";
const Popover = PopoverPrimitive.Root;
const PopoverTrigger = PopoverPrimitive.Trigger;
const PopoverAnchor = PopoverPrimitive.Anchor;
const PopoverContent = React.forwardRef<
React.ElementRef<typeof PopoverPrimitive.Content>,
React.ComponentPropsWithoutRef<typeof PopoverPrimitive.Content>
>(({ className, align = "center", sideOffset = 6, ...props }, ref) => (
<PopoverPrimitive.Portal>
<PopoverPrimitive.Content
ref={ref}
align={align}
sideOffset={sideOffset}
className={cn(
"z-50 w-72 rounded-2xl border border-border bg-popover/95 p-3 text-popover-foreground shadow-xl backdrop-blur-md outline-none",
"data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0 data-[state=closed]:zoom-out-95 data-[state=open]:zoom-in-95",
"data-[side=bottom]:slide-in-from-top-2 data-[side=top]:slide-in-from-bottom-2 data-[side=left]:slide-in-from-right-2 data-[side=right]:slide-in-from-left-2",
className,
)}
{...props}
/>
</PopoverPrimitive.Portal>
));
PopoverContent.displayName = PopoverPrimitive.Content.displayName;
export { Popover, PopoverTrigger, PopoverContent, PopoverAnchor };
@@ -0,0 +1,25 @@
"use client";
import * as React from "react";
import * as SliderPrimitive from "@radix-ui/react-slider";
import { cn } from "@/lib/utils";
const Slider = React.forwardRef<
React.ElementRef<typeof SliderPrimitive.Root>,
React.ComponentPropsWithoutRef<typeof SliderPrimitive.Root>
>(({ className, ...props }, ref) => (
<SliderPrimitive.Root
ref={ref}
className={cn("relative flex w-full touch-none select-none items-center", className)}
{...props}
>
<SliderPrimitive.Track className="relative h-1.5 w-full grow overflow-hidden rounded-full bg-muted">
<SliderPrimitive.Range className="absolute h-full bg-accent-blue" />
</SliderPrimitive.Track>
<SliderPrimitive.Thumb className="block size-4 rounded-full border border-accent-blue/40 bg-background shadow transition-colors focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent-blue/40 disabled:pointer-events-none disabled:opacity-50" />
</SliderPrimitive.Root>
));
Slider.displayName = SliderPrimitive.Root.displayName;
export { Slider };
@@ -0,0 +1,48 @@
"use client";
import * as React from "react";
import * as TabsPrimitive from "@radix-ui/react-tabs";
import { cn } from "@/lib/utils";
const Tabs = TabsPrimitive.Root;
const TabsList = React.forwardRef<
React.ElementRef<typeof TabsPrimitive.List>,
React.ComponentPropsWithoutRef<typeof TabsPrimitive.List>
>(({ className, ...props }, ref) => (
<TabsPrimitive.List
ref={ref}
className={cn("inline-flex items-center gap-1 rounded-full bg-muted/60 p-1 text-muted-foreground", className)}
{...props}
/>
));
TabsList.displayName = TabsPrimitive.List.displayName;
const TabsTrigger = React.forwardRef<
React.ElementRef<typeof TabsPrimitive.Trigger>,
React.ComponentPropsWithoutRef<typeof TabsPrimitive.Trigger>
>(({ className, ...props }, ref) => (
<TabsPrimitive.Trigger
ref={ref}
className={cn(
"inline-flex items-center justify-center rounded-full px-3 py-1.5 text-xs font-medium whitespace-nowrap transition-all",
"focus-visible:outline-none focus-visible:ring-2 focus-visible:ring-accent-blue/40",
"disabled:pointer-events-none disabled:opacity-50",
"data-[state=active]:bg-card data-[state=active]:text-foreground data-[state=active]:shadow-sm",
className,
)}
{...props}
/>
));
TabsTrigger.displayName = TabsPrimitive.Trigger.displayName;
const TabsContent = React.forwardRef<
React.ElementRef<typeof TabsPrimitive.Content>,
React.ComponentPropsWithoutRef<typeof TabsPrimitive.Content>
>(({ className, ...props }, ref) => (
<TabsPrimitive.Content ref={ref} className={cn("mt-4 outline-none", className)} {...props} />
));
TabsContent.displayName = TabsPrimitive.Content.displayName;
export { Tabs, TabsList, TabsTrigger, TabsContent };
@@ -0,0 +1,80 @@
import { describe, expect, it } from "vitest";
import {
DEFAULT_LOBBY_CAPABILITIES_BUNDLE,
clampLobbySelectionToCapabilities,
parseLobbyCapabilitiesBundle,
resolveModelCapabilities,
validateLobbyCreationSelection,
} from "@/lib/creationCapabilities";
describe("creationCapabilities", () => {
it("parses backend capability payloads with per-model caps", () => {
const bundle = parseLobbyCapabilitiesBundle({
model_ids: ["fast-ltx2", "fast-h3"],
models: {
"fast-ltx2": {
generation_modes: ["t2va"],
resolutions: ["480p", "720p"],
duration_sec: [5, 10],
},
"fast-h3": {
generation_modes: ["t2va", "ref2va"],
aspect_ratios: ["16:9"],
resolutions: ["720p"],
},
},
});
expect(bundle.model_ids).toEqual(["fast-ltx2", "fast-h3"]);
expect(bundle.models["fast-h3"]?.aspect_ratios).toEqual(["16:9"]);
});
it("includes fast-h3 in default lobby models", () => {
expect(DEFAULT_LOBBY_CAPABILITIES_BUNDLE.model_ids).toContain("fast-h3");
});
it("clamps unsupported lobby selections to model-specific defaults", () => {
expect(
clampLobbySelectionToCapabilities({
capabilities: resolveModelCapabilities(DEFAULT_LOBBY_CAPABILITIES_BUNDLE, "fast-h3"),
modelId: "fast-h3",
modeId: "fl2av",
aspectRatio: "9:16",
resolution: "4k",
durationSec: 99,
}),
).toEqual({
modelId: "fast-h3",
modeId: "t2v",
aspectRatio: "16:9",
resolution: "720p",
durationSec: 5,
});
});
it("rejects unsupported generation modes with a clear message", () => {
expect(
validateLobbyCreationSelection({
capabilities: resolveModelCapabilities(DEFAULT_LOBBY_CAPABILITIES_BUNDLE, "fast-ltx23"),
modelId: "fast-ltx23",
modeId: "fl2av",
aspectRatio: "16:9",
resolution: "720p",
durationSec: 5,
}),
).toMatch(/FL2VA/i);
});
it("rejects unsupported resolutions for ltx models", () => {
expect(
validateLobbyCreationSelection({
capabilities: resolveModelCapabilities(DEFAULT_LOBBY_CAPABILITIES_BUNDLE, "fast-ltx23"),
modelId: "fast-ltx23",
modeId: "t2v",
aspectRatio: "16:9",
resolution: "4k",
durationSec: 5,
}),
).toMatch(/resolution/i);
});
});
@@ -0,0 +1,263 @@
import type {
AspectRatioId,
CreationModeId,
CreationModelId,
ResolutionId,
} from "@/lib/creationConfig";
import { fromGenerationMode, toGenerationMode, type GenerationMode } from "@/lib/generationMode";
const ALL_MODEL_IDS: CreationModelId[] = ["fast-ltx23", "fast-ltx2", "fast-h3"];
const ALL_GENERATION_MODES: GenerationMode[] = ["t2va", "fl2va", "ref2va"];
const ALL_ASPECT_RATIOS: AspectRatioId[] = ["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"];
const ALL_RESOLUTIONS: ResolutionId[] = ["480p", "720p", "1080p", "4k"];
export interface ModelCreationCapabilities {
generation_modes: GenerationMode[];
aspect_ratios: AspectRatioId[];
resolutions: ResolutionId[];
duration_sec: number[];
unsupported_generation_modes: Record<string, string>;
reference_assets: {
mime_types: string[];
max_bytes: number;
};
}
export interface LobbyCreationCapabilities extends ModelCreationCapabilities {
model_ids: CreationModelId[];
}
export interface LobbyCapabilitiesBundle {
model_ids: CreationModelId[];
models: Partial<Record<CreationModelId, ModelCreationCapabilities>>;
generation_modes: GenerationMode[];
aspect_ratios: AspectRatioId[];
resolutions: ResolutionId[];
duration_sec: number[];
unsupported_generation_modes: Record<string, string>;
reference_assets: {
mime_types: string[];
max_bytes: number;
};
}
const DEFAULT_LTX_MODEL_CAPABILITIES: ModelCreationCapabilities = {
generation_modes: ["t2va", "ref2va"],
aspect_ratios: ["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"],
resolutions: ["480p", "720p", "1080p"],
duration_sec: [5, 10, 15],
unsupported_generation_modes: {
fl2va: "First/last frame mode (FL2VA) is not supported yet.",
},
reference_assets: {
mime_types: ["image/png", "image/jpeg", "image/webp"],
max_bytes: 15 * 1024 * 1024,
},
};
const DEFAULT_H3_MODEL_CAPABILITIES: ModelCreationCapabilities = {
generation_modes: ["t2va", "ref2va"],
aspect_ratios: ["16:9"],
resolutions: ["720p"],
duration_sec: [5, 10, 15],
unsupported_generation_modes: {
fl2va: "First/last frame mode (FL2VA) is not supported yet.",
},
reference_assets: DEFAULT_LTX_MODEL_CAPABILITIES.reference_assets,
};
export const DEFAULT_LOBBY_CAPABILITIES_BUNDLE: LobbyCapabilitiesBundle = {
model_ids: ALL_MODEL_IDS,
models: {
"fast-ltx2": DEFAULT_LTX_MODEL_CAPABILITIES,
"fast-ltx23": DEFAULT_LTX_MODEL_CAPABILITIES,
"fast-h3": DEFAULT_H3_MODEL_CAPABILITIES,
},
generation_modes: ["t2va", "ref2va"],
aspect_ratios: ["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"],
resolutions: ["480p", "720p", "1080p"],
duration_sec: [5, 10, 15],
unsupported_generation_modes: DEFAULT_LTX_MODEL_CAPABILITIES.unsupported_generation_modes,
reference_assets: DEFAULT_LTX_MODEL_CAPABILITIES.reference_assets,
};
function pickStrings<T extends string>(value: unknown, allowed: readonly T[], fallback: readonly T[]): T[] {
if (!Array.isArray(value)) return [...fallback];
return value.filter((item): item is T => typeof item === "string" && allowed.includes(item as T));
}
function parseReferenceAssets(
value: unknown,
fallback: ModelCreationCapabilities["reference_assets"],
): ModelCreationCapabilities["reference_assets"] {
if (!value || typeof value !== "object") return fallback;
const data = value as Record<string, unknown>;
return {
mime_types: Array.isArray(data.mime_types)
? (data.mime_types as string[])
: fallback.mime_types,
max_bytes: typeof data.max_bytes === "number" ? data.max_bytes : fallback.max_bytes,
};
}
function parseModelCreationCapabilities(
value: unknown,
fallback: ModelCreationCapabilities,
): ModelCreationCapabilities {
if (!value || typeof value !== "object") return fallback;
const data = value as Record<string, unknown>;
return {
generation_modes: pickStrings(data.generation_modes, ALL_GENERATION_MODES, fallback.generation_modes),
aspect_ratios: pickStrings(data.aspect_ratios, ALL_ASPECT_RATIOS, fallback.aspect_ratios),
resolutions: pickStrings(data.resolutions, ALL_RESOLUTIONS, fallback.resolutions),
duration_sec: Array.isArray(data.duration_sec)
? data.duration_sec.filter((item): item is number => typeof item === "number")
: fallback.duration_sec,
unsupported_generation_modes:
typeof data.unsupported_generation_modes === "object" && data.unsupported_generation_modes
? (data.unsupported_generation_modes as Record<string, string>)
: fallback.unsupported_generation_modes,
reference_assets: parseReferenceAssets(data.reference_assets, fallback.reference_assets),
};
}
export function parseLobbyCapabilitiesBundle(payload: unknown): LobbyCapabilitiesBundle {
if (!payload || typeof payload !== "object") {
return DEFAULT_LOBBY_CAPABILITIES_BUNDLE;
}
const data = payload as Record<string, unknown>;
const modelIds = pickStrings(data.model_ids, ALL_MODEL_IDS, DEFAULT_LOBBY_CAPABILITIES_BUNDLE.model_ids);
const rawModels = typeof data.models === "object" && data.models ? (data.models as Record<string, unknown>) : {};
const models: Partial<Record<CreationModelId, ModelCreationCapabilities>> = {};
for (const modelId of modelIds) {
const fallback =
DEFAULT_LOBBY_CAPABILITIES_BUNDLE.models[modelId] ??
(modelId === "fast-h3" ? DEFAULT_H3_MODEL_CAPABILITIES : DEFAULT_LTX_MODEL_CAPABILITIES);
models[modelId] = parseModelCreationCapabilities(rawModels[modelId], fallback);
}
const unionFallback = parseModelCreationCapabilities(payload, DEFAULT_LTX_MODEL_CAPABILITIES);
return {
model_ids: modelIds,
models,
generation_modes: unionFallback.generation_modes,
aspect_ratios: unionFallback.aspect_ratios,
resolutions: unionFallback.resolutions,
duration_sec: unionFallback.duration_sec,
unsupported_generation_modes: unionFallback.unsupported_generation_modes,
reference_assets: unionFallback.reference_assets,
};
}
export function resolveModelCapabilities(
bundle: LobbyCapabilitiesBundle,
modelId: CreationModelId,
): LobbyCreationCapabilities {
const modelCaps =
bundle.models[modelId] ??
(modelId === "fast-h3" ? DEFAULT_H3_MODEL_CAPABILITIES : DEFAULT_LTX_MODEL_CAPABILITIES);
return {
model_ids: bundle.model_ids,
...modelCaps,
};
}
export function supportedCreationModes(capabilities: LobbyCreationCapabilities) {
return capabilities.generation_modes.map((wireMode) => ({
wireMode,
modeId: fromGenerationMode(wireMode),
}));
}
export function isSupportedCreationMode(modeId: CreationModeId, capabilities: LobbyCreationCapabilities): boolean {
return capabilities.generation_modes.includes(toGenerationMode(modeId));
}
export function isSupportedResolution(resolution: ResolutionId, capabilities: LobbyCreationCapabilities): boolean {
return capabilities.resolutions.includes(resolution);
}
export function isSupportedReferenceImage(file: File, capabilities: LobbyCreationCapabilities): boolean {
return capabilities.reference_assets.mime_types.includes(file.type);
}
export function unsupportedModeNotice(modeId: CreationModeId, capabilities: LobbyCreationCapabilities): string | null {
const wireMode = toGenerationMode(modeId);
return capabilities.unsupported_generation_modes[wireMode] ?? null;
}
export function clampLobbySelectionToCapabilities(input: {
capabilities: LobbyCreationCapabilities;
modelId: CreationModelId;
modeId: CreationModeId;
aspectRatio: AspectRatioId;
resolution: ResolutionId;
durationSec: number;
}): {
modelId: CreationModelId;
modeId: CreationModeId;
aspectRatio: AspectRatioId;
resolution: ResolutionId;
durationSec: number;
} {
const { capabilities } = input;
const modelId = capabilities.model_ids.includes(input.modelId)
? input.modelId
: (capabilities.model_ids[0] ?? "fast-ltx23");
const supportedModes = supportedCreationModes(capabilities);
const modeId = isSupportedCreationMode(input.modeId, capabilities)
? input.modeId
: (supportedModes[0]?.modeId ?? "t2v");
const aspectRatio = capabilities.aspect_ratios.includes(input.aspectRatio)
? input.aspectRatio
: (capabilities.aspect_ratios[0] ?? "16:9");
const resolution = isSupportedResolution(input.resolution, capabilities)
? input.resolution
: (capabilities.resolutions[0] ?? "720p");
const durationSec = capabilities.duration_sec.includes(input.durationSec)
? input.durationSec
: (capabilities.duration_sec[0] ?? 5);
return { modelId, modeId, aspectRatio, resolution, durationSec };
}
export function validateLobbyCreationSelection(input: {
capabilities: LobbyCreationCapabilities;
modelId: CreationModelId;
modeId: CreationModeId;
aspectRatio: AspectRatioId;
resolution: ResolutionId;
durationSec: number;
referenceFile?: File | null;
firstFrameFile?: File | null;
lastFrameFile?: File | null;
}): string | null {
const unsupportedMode = unsupportedModeNotice(input.modeId, input.capabilities);
if (unsupportedMode) return unsupportedMode;
if (!input.capabilities.model_ids.includes(input.modelId)) {
return "Selected model is not supported yet.";
}
if (!isSupportedCreationMode(input.modeId, input.capabilities)) {
return "Selected mode is not supported yet.";
}
if (!input.capabilities.aspect_ratios.includes(input.aspectRatio)) {
return "Selected aspect ratio is not supported for this model yet.";
}
if (!isSupportedResolution(input.resolution, input.capabilities)) {
return "Selected resolution is not supported for this model yet.";
}
if (!input.capabilities.duration_sec.includes(input.durationSec)) {
return "Selected duration is not supported yet.";
}
if (input.modeId === "ref2av" && !input.referenceFile) {
return "Upload a reference image to use reference-guided mode.";
}
if (input.referenceFile && !isSupportedReferenceImage(input.referenceFile, input.capabilities)) {
return "Reference assets must be PNG, JPEG, or WebP images.";
}
if (input.firstFrameFile && !isSupportedReferenceImage(input.firstFrameFile, input.capabilities)) {
return "First frame must be a PNG, JPEG, or WebP image.";
}
if (input.lastFrameFile && !isSupportedReferenceImage(input.lastFrameFile, input.capabilities)) {
return "Last frame must be a PNG, JPEG, or WebP image.";
}
return null;
}
@@ -0,0 +1,62 @@
import { describe, expect, it } from "vitest";
import {
CREATION_MODELS,
buildMentionOptions,
formatDurationLabel,
formatResolutionLabel,
isReferenceMediaFile,
modeRequiresReference,
modeUsesDualFrames,
} from "@/lib/creationConfig";
describe("creationConfig", () => {
it("formats resolution labels", () => {
expect(formatResolutionLabel("480p")).toBe("480P");
expect(formatResolutionLabel("720p")).toBe("720P");
expect(formatResolutionLabel("4k")).toBe("4K");
});
it("formats duration labels", () => {
expect(formatDurationLabel(5)).toBe("5s");
});
it("includes all Dreamverse lobby models", () => {
expect(CREATION_MODELS.map((model) => model.id)).toEqual(["fast-ltx23", "fast-ltx2", "fast-h3"]);
});
it("builds mention options from presets", () => {
expect(
buildMentionOptions([
{ id: "preset-a", label: "Preset A", description: "A short preset" },
{ label: "Missing id" },
]),
).toEqual([
{
id: "preset-a",
label: "Preset A",
kind: "preset",
description: "A short preset",
},
{
id: "Missing id",
label: "Missing id",
kind: "preset",
description: undefined,
},
]);
});
it("derives mode-specific reference requirements", () => {
expect(modeRequiresReference("ref2av")).toBe(true);
expect(modeRequiresReference("t2v")).toBe(false);
expect(modeUsesDualFrames("fl2av")).toBe(true);
expect(modeUsesDualFrames("t2v")).toBe(false);
});
it("accepts image reference files only", () => {
expect(isReferenceMediaFile(new File(["x"], "a.png", { type: "image/png" }))).toBe(true);
expect(isReferenceMediaFile(new File(["x"], "a.mp4", { type: "video/mp4" }))).toBe(false);
expect(isReferenceMediaFile(new File(["x"], "a.txt", { type: "text/plain" }))).toBe(false);
});
});
@@ -0,0 +1,96 @@
export type CreationModeId = "t2v" | "fl2av" | "ref2av";
export type CreationModelId = "fast-ltx2" | "fast-ltx23" | "fast-h3";
export type AspectRatioId = "21:9" | "16:9" | "4:3" | "1:1" | "3:4" | "9:16";
export type ResolutionId = "480p" | "720p" | "1080p" | "4k";
export interface CreationModeOption {
id: CreationModeId;
label: string;
description: string;
}
export interface CreationModelOption {
id: CreationModelId;
label: string;
description: string;
badge?: string;
}
export interface MentionOption {
id: string;
label: string;
kind: "preset" | "asset" | "character";
description?: string;
}
export const CREATION_MODES: CreationModeOption[] = [
{ id: "t2v", label: "Text to video", description: "Generate from a text prompt" },
{ id: "ref2av", label: "Image to video", description: "Guide the first segment with a reference image" },
];
export const UNSUPPORTED_CREATION_MODES: CreationModeOption[] = [
{ id: "fl2av", label: "First and last frame", description: "Coming soon on FastLTX models" },
];
export const CREATION_MODELS: CreationModelOption[] = [
{
id: "fast-ltx23",
label: "FastLTX 2.3",
description: "LTX 2.3 with OmniNFT LoRA",
badge: "New",
},
{
id: "fast-ltx2",
label: "FastLTX 2",
description: "FastLTX 2 for streaming",
},
{
id: "fast-h3",
label: "FastH3",
description: "MiniMax H3 with VSA data-free adapter",
},
];
export const ASPECT_RATIOS: AspectRatioId[] = ["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"];
export const RESOLUTIONS: ResolutionId[] = ["480p", "720p", "1080p"];
export const UNSUPPORTED_RESOLUTIONS: ResolutionId[] = ["4k"];
export const DURATION_MARKS = [5, 10, 15] as const;
export const REFERENCE_ACCEPT = "image/png,image/jpeg,image/webp";
export function formatResolutionLabel(resolution: ResolutionId): string {
return resolution === "4k" ? "4K" : resolution.toUpperCase();
}
export function formatDurationLabel(seconds: number): string {
return `${seconds}s`;
}
export function modeRequiresReference(modeId: CreationModeId): boolean {
return modeId === "ref2av";
}
export function modeUsesDualFrames(modeId: CreationModeId): boolean {
return modeId === "fl2av";
}
export function isReferenceMediaFile(file: File): boolean {
return file.type === "image/png" || file.type === "image/jpeg" || file.type === "image/webp";
}
export function buildMentionOptions(storyPresets: Array<{ id?: string; label?: string; description?: string }>): MentionOption[] {
return storyPresets
.filter((preset) => typeof preset.label === "string" && preset.label.trim())
.map((preset) => ({
id: String(preset.id || preset.label),
label: String(preset.label),
kind: "preset" as const,
description: typeof preset.description === "string" ? preset.description : undefined,
}));
}
@@ -0,0 +1,66 @@
import { describe, expect, it } from "vitest";
import { parseEchoedCreationConfig, validateCreationInputs } from "@/lib/creationPayload";
describe("creationPayload", () => {
it("requires a reference asset for omni reference mode", () => {
expect(
validateCreationInputs({
modeId: "ref2av",
referenceFile: null,
}),
).toMatch(/reference asset/i);
});
it("requires both frames for first and last frame mode", () => {
expect(
validateCreationInputs({
modeId: "fl2av",
firstFrameFile: new File(["a"], "first.png", { type: "image/png" }),
lastFrameFile: null,
}),
).toMatch(/both first and last/i);
});
it("accepts text to video without references", () => {
expect(
validateCreationInputs({
modeId: "t2v",
}),
).toBeNull();
});
it("parses echoed creation config from server payloads", () => {
expect(
parseEchoedCreationConfig({
type: "gpu_assigned",
creation_config: {
model_id: "fast-ltx2",
generation_mode: "ref2va",
aspect_ratio: "9:16",
resolution: "480p",
duration_sec: 10,
},
}),
).toEqual({
modelId: "fast-ltx2",
modeId: "ref2av",
aspectRatio: "9:16",
resolution: "480p",
durationSec: 10,
});
});
it("ignores invalid echoed creation config", () => {
expect(parseEchoedCreationConfig({ creation_config: { model_id: "unknown" } })).toBeNull();
});
it("rejects unsupported reference mime types", () => {
expect(
validateCreationInputs({
modeId: "t2v",
referenceFile: new File(["a"], "clip.mp4", { type: "video/mp4" }),
}),
).toMatch(/PNG, JPEG, or WebP/i);
});
});
@@ -0,0 +1,172 @@
import type {
AspectRatioId,
CreationModeId,
CreationModelId,
ResolutionId,
} from "@/lib/creationConfig";
import { fromGenerationMode, type GenerationMode } from "@/lib/generationMode";
const LOBBY_MODEL_IDS = new Set<CreationModelId>(["fast-ltx2", "fast-ltx23", "fast-h3"]);
const ASPECT_RATIO_IDS = new Set<AspectRatioId>(["21:9", "16:9", "4:3", "1:1", "3:4", "9:16"]);
const RESOLUTION_IDS = new Set<ResolutionId>(["480p", "720p", "1080p", "4k"]);
const DURATION_SEC_VALUES = new Set([5, 10, 15]);
export interface EchoedSessionCreationConfig {
modelId: CreationModelId;
modeId: CreationModeId;
aspectRatio: AspectRatioId;
resolution: ResolutionId;
durationSec: number;
}
const MAX_IMAGE_BYTES = 15 * 1024 * 1024;
const SUPPORTED_IMAGE_TYPES = new Set(["image/png", "image/jpeg", "image/webp"]);
export interface InitialImagePayload {
name: string;
mime_type: string;
data_url: string;
}
export interface CreationInitPayload {
model_id: string;
aspect_ratio: string;
resolution: string;
duration_sec: number;
initial_image: InitialImagePayload | null;
last_frame_image: InitialImagePayload | null;
}
function readFileAsDataUrl(file: File): Promise<string> {
return new Promise((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
if (typeof reader.result === "string") {
resolve(reader.result);
return;
}
reject(new Error("Failed to read reference image."));
};
reader.onerror = () => reject(new Error("Failed to read reference image."));
reader.readAsDataURL(file);
});
}
export async function fileToInitialImagePayload(file: File): Promise<InitialImagePayload> {
if (!SUPPORTED_IMAGE_TYPES.has(file.type)) {
throw new Error("Reference assets must be PNG, JPEG, or WebP images.");
}
if (file.size > MAX_IMAGE_BYTES) {
throw new Error("Reference image must be 15 MB or smaller.");
}
return {
name: file.name,
mime_type: file.type,
data_url: await readFileAsDataUrl(file),
};
}
export async function resolveCreationImages(input: {
modeId: CreationModeId;
referenceFile?: File | null;
firstFrameFile?: File | null;
lastFrameFile?: File | null;
}): Promise<Pick<CreationInitPayload, "initial_image" | "last_frame_image">> {
if (input.modeId === "fl2av") {
const firstFrame = input.firstFrameFile ? await fileToInitialImagePayload(input.firstFrameFile) : null;
const lastFrame = input.lastFrameFile ? await fileToInitialImagePayload(input.lastFrameFile) : null;
return {
initial_image: firstFrame,
last_frame_image: lastFrame,
};
}
const reference = input.referenceFile ? await fileToInitialImagePayload(input.referenceFile) : null;
return {
initial_image: reference,
last_frame_image: null,
};
}
export function validateCreationInputs(input: {
modeId: CreationModeId;
referenceFile?: File | null;
firstFrameFile?: File | null;
lastFrameFile?: File | null;
}): string | null {
if (input.modeId === "ref2av" && !input.referenceFile) {
return "Upload a reference asset to use Omni reference mode.";
}
if (input.modeId === "fl2av") {
if (!input.firstFrameFile || !input.lastFrameFile) {
return "Upload both first and last frame assets.";
}
}
if (input.referenceFile && !SUPPORTED_IMAGE_TYPES.has(input.referenceFile.type)) {
return "Reference assets must be PNG, JPEG, or WebP images.";
}
if (input.firstFrameFile && !SUPPORTED_IMAGE_TYPES.has(input.firstFrameFile.type)) {
return "First frame must be a PNG, JPEG, or WebP image.";
}
if (input.lastFrameFile && !SUPPORTED_IMAGE_TYPES.has(input.lastFrameFile.type)) {
return "Last frame must be a PNG, JPEG, or WebP image.";
}
return null;
}
export function parseEchoedCreationConfig(data: unknown): EchoedSessionCreationConfig | null {
if (!data || typeof data !== "object") {
return null;
}
const creationConfig = (data as Record<string, unknown>).creation_config;
if (!creationConfig || typeof creationConfig !== "object") {
return null;
}
const config = creationConfig as Record<string, unknown>;
const modelId = typeof config.model_id === "string" && LOBBY_MODEL_IDS.has(config.model_id as CreationModelId)
? (config.model_id as CreationModelId)
: null;
const generationMode = typeof config.generation_mode === "string" ? config.generation_mode as GenerationMode : null;
const modeId = generationMode === "t2va" || generationMode === "fl2va" || generationMode === "ref2va"
? fromGenerationMode(generationMode)
: null;
const aspectRatio = typeof config.aspect_ratio === "string" && ASPECT_RATIO_IDS.has(config.aspect_ratio as AspectRatioId)
? (config.aspect_ratio as AspectRatioId)
: null;
const resolution = typeof config.resolution === "string" && RESOLUTION_IDS.has(config.resolution as ResolutionId)
? (config.resolution as ResolutionId)
: null;
const durationSec = typeof config.duration_sec === "number" && DURATION_SEC_VALUES.has(config.duration_sec)
? config.duration_sec
: null;
if (modelId === null || modeId === null || aspectRatio === null || resolution === null || durationSec === null) {
return null;
}
return {
modelId,
modeId,
aspectRatio,
resolution,
durationSec,
};
}
export async function buildCreationInitPayload(input: {
modelId: string;
modeId: CreationModeId;
aspectRatio: string;
resolution: string;
durationSec: number;
referenceFile?: File | null;
firstFrameFile?: File | null;
lastFrameFile?: File | null;
}): Promise<CreationInitPayload> {
const images = await resolveCreationImages(input);
return {
model_id: input.modelId,
aspect_ratio: input.aspectRatio,
resolution: input.resolution,
duration_sec: input.durationSec,
...images,
};
}
@@ -0,0 +1,39 @@
import { describe, expect, it } from "vitest";
import {
DEFAULT_GENERATION_MODE,
GENERATION_MODES,
fromGenerationMode,
getGenerationMode,
isGenerationMode,
toGenerationMode,
} from "./generationMode";
describe("generation modes", () => {
it("exposes stable wire IDs in the expected product order", () => {
expect(GENERATION_MODES.map((mode) => mode.id)).toEqual([
"t2va",
"fl2va",
"ref2va",
]);
expect(DEFAULT_GENERATION_MODE).toBe("t2va");
});
it("validates and resolves generation mode values", () => {
expect(isGenerationMode("ref2va")).toBe(true);
expect(isGenerationMode("unknown")).toBe(false);
expect(getGenerationMode("fl2va").label).toBe("FL2VA");
});
it("maps creation studio mode IDs to upstream wire values", () => {
expect(toGenerationMode("t2v")).toBe("t2va");
expect(toGenerationMode("fl2av")).toBe("fl2va");
expect(toGenerationMode("ref2av")).toBe("ref2va");
});
it("maps upstream wire values back to creation studio mode IDs", () => {
expect(fromGenerationMode("t2va")).toBe("t2v");
expect(fromGenerationMode("fl2va")).toBe("fl2av");
expect(fromGenerationMode("ref2va")).toBe("ref2av");
});
});
@@ -0,0 +1,54 @@
import type { CreationModeId } from "@/lib/creationConfig";
export const GENERATION_MODES = [
{
id: "t2va",
label: "T2VA",
name: "Text to video + audio",
description: "Start with a text prompt; no reference asset is required.",
},
{
id: "fl2va",
label: "FL2VA",
name: "First/last frames to video + audio",
description: "Provide first and last frame images to control the transition.",
},
{
id: "ref2va",
label: "Ref2VA",
name: "References to video + audio",
description: "Guide the result with ordered image, video, or audio references.",
},
] as const;
export type GenerationMode = (typeof GENERATION_MODES)[number]["id"];
export const DEFAULT_GENERATION_MODE: GenerationMode = "t2va";
const CREATION_MODE_TO_GENERATION_MODE: Record<CreationModeId, GenerationMode> = {
t2v: "t2va",
fl2av: "fl2va",
ref2av: "ref2va",
};
export function isGenerationMode(value: unknown): value is GenerationMode {
return GENERATION_MODES.some((mode) => mode.id === value);
}
export function getGenerationMode(value: GenerationMode) {
return GENERATION_MODES.find((mode) => mode.id === value) ?? GENERATION_MODES[0];
}
const GENERATION_MODE_TO_CREATION_MODE: Record<GenerationMode, CreationModeId> = {
t2va: "t2v",
fl2va: "fl2av",
ref2va: "ref2av",
};
export function fromGenerationMode(mode: GenerationMode): CreationModeId {
return GENERATION_MODE_TO_CREATION_MODE[mode];
}
export function toGenerationMode(modeId: CreationModeId): GenerationMode {
return CREATION_MODE_TO_GENERATION_MODE[modeId];
}