[fix] dreamverse: env-tunable session timeout, restore warmup watchdog, derive SP size in launch script

- SESSION_TIMEOUT_SECONDS: default back to 300 and now reads the
  DREAMVERSE_SESSION_TIMEOUT_SECONDS env var that launch-dreamverse.sh was
  already exporting (previously nothing read it, and the hardcoded 1800 made
  every deployment hold idle GPU-pool slots 6x longer). Fixes the stale
  five-minute-timeout test and adds an override test.
- STARTUP_WARMUP_TIMEOUT_SECONDS: default back to 2400 so the warmup watchdog
  works again; launch-dreamverse.sh exports the existing
  FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS override (24000) for the slow
  GB200 max-autotune boot.
- launch-dreamverse.sh: derive the DREAMVERSE_SP_SIZE default from the number
  of visible GPUs so the documented CUDA_VISIBLE_DEVICES=0 invocation no
  longer crashes gpu_pool with 'Not enough GPUs'; explicit env still wins.
  setup-dreamverse-env.sh's printed instruction now matches. gpu_pool raises
  a friendly error for non-integer CUDA_VISIBLE_DEVICES entries.
- video_generation.py: drop unused ParallelismConfig import (F401).
This commit is contained in:
SolitaryThinker
2026-07-16 20:39:13 -07:00
parent deb51f1dcc
commit cfb54a3b2a
7 changed files with 40 additions and 12 deletions
+4 -4
View File
@@ -76,9 +76,6 @@ if ACTIVE_MODEL_ID not in MODEL_REGISTRY:
# Active model configuration
MODEL_CONFIG = MODEL_REGISTRY[ACTIVE_MODEL_ID]
# Generation limits
SESSION_TIMEOUT_SECONDS = 1800
# Frame settings
NUM_FRAMES = 121
FRAME_HEIGHT = 1088
@@ -168,6 +165,9 @@ def _optional_env(*names: str) -> str | None:
return None
# Generation limits
SESSION_TIMEOUT_SECONDS = _env_int("DREAMVERSE_SESSION_TIMEOUT_SECONDS", 300)
DEVTOOLS_ENABLED = _env_bool("FASTVIDEO_ENABLE_DEVTOOLS", False)
PROMPT_SAFETY_ENABLED = _env_bool("FASTVIDEO_ENABLE_PROMPT_SAFETY", False)
DREAMVERSE_MAX_AUTOTUNE = _env_bool("DREAMVERSE_MAX_AUTOTUNE", True)
@@ -376,4 +376,4 @@ STARTUP_WARMUP_PROMPT = os.getenv(
("A cinematic drone shot over coastal cliffs at sunrise, "
"golden light, gentle ocean waves, ultra detailed"),
).strip()
STARTUP_WARMUP_TIMEOUT_SECONDS = max(1, _env_int("FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS", 24000))
STARTUP_WARMUP_TIMEOUT_SECONDS = max(1, _env_int("FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS", 2400))
+5 -1
View File
@@ -1023,7 +1023,11 @@ def get_available_gpus() -> list[int]:
"""Get list of available GPU IDs from environment or auto-detect."""
cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES", "")
if cuda_visible:
visible_gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()]
try:
visible_gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()]
except ValueError as exc:
raise RuntimeError("CUDA_VISIBLE_DEVICES must be a comma-separated list of integer GPU "
f"indices (got {cuda_visible!r}); GPU UUIDs are not supported.") from exc
return _limit_gpu_ids(visible_gpu_ids)
# Auto-detect available GPUs
@@ -150,12 +150,22 @@ def test_config_enables_prompt_safety_when_requested(monkeypatch):
def test_config_uses_five_minute_session_timeout(monkeypatch):
_set_required_prompt_keys(monkeypatch)
monkeypatch.delenv("DREAMVERSE_SESSION_TIMEOUT_SECONDS", raising=False)
module = _load_config_module()
assert module.SESSION_TIMEOUT_SECONDS == 300
def test_config_session_timeout_env_override(monkeypatch):
_set_required_prompt_keys(monkeypatch)
monkeypatch.setenv("DREAMVERSE_SESSION_TIMEOUT_SECONDS", "1800")
module = _load_config_module()
assert module.SESSION_TIMEOUT_SECONDS == 1800
def test_config_rejects_invalid_prompt_provider(monkeypatch):
monkeypatch.setenv("FASTVIDEO_PROMPT_PROVIDER", "unsupported")
_set_required_prompt_keys(monkeypatch)
@@ -265,7 +265,6 @@ class VideoGenerationWorker:
EngineConfig,
GeneratorConfig,
OffloadConfig,
ParallelismConfig,
PipelineSelection,
QuantizationConfig,
)
+18 -3
View File
@@ -2,7 +2,8 @@
# launch-dreamverse.sh — launch dreamverse-server on a compute node.
#
# Usage (from repo root):
# CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh
# bash apps/dreamverse/scripts/launch-dreamverse.sh # GPUs 0-3, SP_SIZE=4
# CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh # single GPU, SP_SIZE=1
set -euo pipefail
@@ -18,7 +19,9 @@ export DREAMVERSE_MAX_AUTOTUNE=true
export LTX2_USE_DISTILLED_SIGMAS=0
export LTX2_VIDEO_CONDITIONING_NUM_FRAMES=1
export AUDIO_CONDITIONING_NUM_FRAMES=41
export DREAMVERSE_SESSION_TIMEOUT_SECONDS=1800
export DREAMVERSE_SESSION_TIMEOUT_SECONDS="${DREAMVERSE_SESSION_TIMEOUT_SECONDS:-1800}"
# GB200 max-autotune warmup compiles can run for hours; keep the watchdog generous here
export FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS="${FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS:-24000}"
export CEREBRAS_API_KEY="${CEREBRAS_API_KEY:-}" # set this in your env or ~/.env
export FASTVIDEO_PROMPT_CEREBRAS_MODEL="gpt-oss-120b"
export TORCHINDUCTOR_CACHE_DIR="$HOME/.cache/torchinductor"
@@ -40,7 +43,19 @@ export LD_LIBRARY_PATH="$CUDA_RT_DIR"
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}"
export FASTVIDEO_GPU_COUNT="${FASTVIDEO_GPU_COUNT:-all}"
export DREAMVERSE_SP_SIZE="${DREAMVERSE_SP_SIZE:-4}"
# Default SP size to the usable GPU count so single-GPU invocations work.
# A numeric FASTVIDEO_GPU_COUNT caps the pool below the visible count, and an
# SP size above the pool size fails GPUPool startup with "Not enough GPUs".
IFS=',' read -ra _VISIBLE_GPUS <<< "$CUDA_VISIBLE_DEVICES"
# Count only non-empty tokens, matching gpu_pool.get_available_gpus (e.g. ",0,1" is 2 GPUs).
_USABLE_GPU_COUNT=0
for _gpu in "${_VISIBLE_GPUS[@]}"; do
[[ -n "${_gpu//[[:space:]]/}" ]] && _USABLE_GPU_COUNT=$((_USABLE_GPU_COUNT + 1))
done
if [[ "$FASTVIDEO_GPU_COUNT" =~ ^[0-9]+$ ]] && (( FASTVIDEO_GPU_COUNT < _USABLE_GPU_COUNT )); then
_USABLE_GPU_COUNT="$FASTVIDEO_GPU_COUNT"
fi
export DREAMVERSE_SP_SIZE="${DREAMVERSE_SP_SIZE:-$_USABLE_GPU_COUNT}"
PORT="${DREAMVERSE_PORT:-8009}"
FFMPEG_ENV="$(dirname "$0")/ffmpeg-env.sh"
@@ -101,5 +101,5 @@ UV_CACHE_DIR="$LOCAL_DIR/cache" "$CONDA_PREFIX/bin/uv" pip install \
https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.11.post3/flashinfer_python-0.6.11.post3-py3-none-any.whl
echo ""
echo "Done. On a compute node run:"
echo " CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh"
echo "Done. On a compute node run (GPUs 0-3 by default; set CUDA_VISIBLE_DEVICES to restrict):"
echo " bash apps/dreamverse/scripts/launch-dreamverse.sh"
@@ -110,7 +110,7 @@ default_request:
fps: 24 # internal: gpu_pool.py:85 TARGET_FPS
streaming:
# internal: config.py:33 SESSION_TIMEOUT_SECONDS = 300
# internal: config.py SESSION_TIMEOUT_SECONDS (env DREAMVERSE_SESSION_TIMEOUT_SECONDS, default 300)
session_timeout_seconds: 300
# internal: config.py:282-284 GENERATION_SEGMENT_CAP default 6
generation_segment_cap: 6