From cfb54a3b2a19cd8efc93d0e55fa6f9ccbd4799cf Mon Sep 17 00:00:00 2001 From: SolitaryThinker Date: Thu, 16 Jul 2026 20:39:13 -0700 Subject: [PATCH] [fix] dreamverse: env-tunable session timeout, restore warmup watchdog, derive SP size in launch script - SESSION_TIMEOUT_SECONDS: default back to 300 and now reads the DREAMVERSE_SESSION_TIMEOUT_SECONDS env var that launch-dreamverse.sh was already exporting (previously nothing read it, and the hardcoded 1800 made every deployment hold idle GPU-pool slots 6x longer). Fixes the stale five-minute-timeout test and adds an override test. - STARTUP_WARMUP_TIMEOUT_SECONDS: default back to 2400 so the warmup watchdog works again; launch-dreamverse.sh exports the existing FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS override (24000) for the slow GB200 max-autotune boot. - launch-dreamverse.sh: derive the DREAMVERSE_SP_SIZE default from the number of visible GPUs so the documented CUDA_VISIBLE_DEVICES=0 invocation no longer crashes gpu_pool with 'Not enough GPUs'; explicit env still wins. setup-dreamverse-env.sh's printed instruction now matches. gpu_pool raises a friendly error for non-integer CUDA_VISIBLE_DEVICES entries. - video_generation.py: drop unused ParallelismConfig import (F401). --- apps/dreamverse/dreamverse/config.py | 8 +++---- apps/dreamverse/dreamverse/gpu_pool.py | 6 +++++- .../dreamverse/tests/test_config.py | 10 +++++++++ .../dreamverse/dreamverse/video_generation.py | 1 - apps/dreamverse/scripts/launch-dreamverse.sh | 21 ++++++++++++++++--- .../scripts/setup-dreamverse-env.sh | 4 ++-- .../serve_configs/streaming_demo.yaml | 2 +- 7 files changed, 40 insertions(+), 12 deletions(-) diff --git a/apps/dreamverse/dreamverse/config.py b/apps/dreamverse/dreamverse/config.py index 3c4402d74..1f14aa678 100644 --- a/apps/dreamverse/dreamverse/config.py +++ b/apps/dreamverse/dreamverse/config.py @@ -76,9 +76,6 @@ if ACTIVE_MODEL_ID not in MODEL_REGISTRY: # Active model configuration MODEL_CONFIG = MODEL_REGISTRY[ACTIVE_MODEL_ID] -# Generation limits -SESSION_TIMEOUT_SECONDS = 1800 - # Frame settings NUM_FRAMES = 121 FRAME_HEIGHT = 1088 @@ -168,6 +165,9 @@ def _optional_env(*names: str) -> str | None: return None +# Generation limits +SESSION_TIMEOUT_SECONDS = _env_int("DREAMVERSE_SESSION_TIMEOUT_SECONDS", 300) + DEVTOOLS_ENABLED = _env_bool("FASTVIDEO_ENABLE_DEVTOOLS", False) PROMPT_SAFETY_ENABLED = _env_bool("FASTVIDEO_ENABLE_PROMPT_SAFETY", False) DREAMVERSE_MAX_AUTOTUNE = _env_bool("DREAMVERSE_MAX_AUTOTUNE", True) @@ -376,4 +376,4 @@ STARTUP_WARMUP_PROMPT = os.getenv( ("A cinematic drone shot over coastal cliffs at sunrise, " "golden light, gentle ocean waves, ultra detailed"), ).strip() -STARTUP_WARMUP_TIMEOUT_SECONDS = max(1, _env_int("FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS", 24000)) +STARTUP_WARMUP_TIMEOUT_SECONDS = max(1, _env_int("FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS", 2400)) diff --git a/apps/dreamverse/dreamverse/gpu_pool.py b/apps/dreamverse/dreamverse/gpu_pool.py index 3a3b98e17..2deff88e0 100644 --- a/apps/dreamverse/dreamverse/gpu_pool.py +++ b/apps/dreamverse/dreamverse/gpu_pool.py @@ -1023,7 +1023,11 @@ def get_available_gpus() -> list[int]: """Get list of available GPU IDs from environment or auto-detect.""" cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES", "") if cuda_visible: - visible_gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()] + try: + visible_gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()] + except ValueError as exc: + raise RuntimeError("CUDA_VISIBLE_DEVICES must be a comma-separated list of integer GPU " + f"indices (got {cuda_visible!r}); GPU UUIDs are not supported.") from exc return _limit_gpu_ids(visible_gpu_ids) # Auto-detect available GPUs diff --git a/apps/dreamverse/dreamverse/tests/test_config.py b/apps/dreamverse/dreamverse/tests/test_config.py index 99a7a0893..872b01dcf 100644 --- a/apps/dreamverse/dreamverse/tests/test_config.py +++ b/apps/dreamverse/dreamverse/tests/test_config.py @@ -150,12 +150,22 @@ def test_config_enables_prompt_safety_when_requested(monkeypatch): def test_config_uses_five_minute_session_timeout(monkeypatch): _set_required_prompt_keys(monkeypatch) + monkeypatch.delenv("DREAMVERSE_SESSION_TIMEOUT_SECONDS", raising=False) module = _load_config_module() assert module.SESSION_TIMEOUT_SECONDS == 300 +def test_config_session_timeout_env_override(monkeypatch): + _set_required_prompt_keys(monkeypatch) + monkeypatch.setenv("DREAMVERSE_SESSION_TIMEOUT_SECONDS", "1800") + + module = _load_config_module() + + assert module.SESSION_TIMEOUT_SECONDS == 1800 + + def test_config_rejects_invalid_prompt_provider(monkeypatch): monkeypatch.setenv("FASTVIDEO_PROMPT_PROVIDER", "unsupported") _set_required_prompt_keys(monkeypatch) diff --git a/apps/dreamverse/dreamverse/video_generation.py b/apps/dreamverse/dreamverse/video_generation.py index be7f5eb63..1a24ee990 100644 --- a/apps/dreamverse/dreamverse/video_generation.py +++ b/apps/dreamverse/dreamverse/video_generation.py @@ -265,7 +265,6 @@ class VideoGenerationWorker: EngineConfig, GeneratorConfig, OffloadConfig, - ParallelismConfig, PipelineSelection, QuantizationConfig, ) diff --git a/apps/dreamverse/scripts/launch-dreamverse.sh b/apps/dreamverse/scripts/launch-dreamverse.sh index dc3e4bd47..33e9ceabe 100755 --- a/apps/dreamverse/scripts/launch-dreamverse.sh +++ b/apps/dreamverse/scripts/launch-dreamverse.sh @@ -2,7 +2,8 @@ # launch-dreamverse.sh — launch dreamverse-server on a compute node. # # Usage (from repo root): -# CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh +# bash apps/dreamverse/scripts/launch-dreamverse.sh # GPUs 0-3, SP_SIZE=4 +# CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh # single GPU, SP_SIZE=1 set -euo pipefail @@ -18,7 +19,9 @@ export DREAMVERSE_MAX_AUTOTUNE=true export LTX2_USE_DISTILLED_SIGMAS=0 export LTX2_VIDEO_CONDITIONING_NUM_FRAMES=1 export AUDIO_CONDITIONING_NUM_FRAMES=41 -export DREAMVERSE_SESSION_TIMEOUT_SECONDS=1800 +export DREAMVERSE_SESSION_TIMEOUT_SECONDS="${DREAMVERSE_SESSION_TIMEOUT_SECONDS:-1800}" +# GB200 max-autotune warmup compiles can run for hours; keep the watchdog generous here +export FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS="${FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS:-24000}" export CEREBRAS_API_KEY="${CEREBRAS_API_KEY:-}" # set this in your env or ~/.env export FASTVIDEO_PROMPT_CEREBRAS_MODEL="gpt-oss-120b" export TORCHINDUCTOR_CACHE_DIR="$HOME/.cache/torchinductor" @@ -40,7 +43,19 @@ export LD_LIBRARY_PATH="$CUDA_RT_DIR" CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" export FASTVIDEO_GPU_COUNT="${FASTVIDEO_GPU_COUNT:-all}" -export DREAMVERSE_SP_SIZE="${DREAMVERSE_SP_SIZE:-4}" +# Default SP size to the usable GPU count so single-GPU invocations work. +# A numeric FASTVIDEO_GPU_COUNT caps the pool below the visible count, and an +# SP size above the pool size fails GPUPool startup with "Not enough GPUs". +IFS=',' read -ra _VISIBLE_GPUS <<< "$CUDA_VISIBLE_DEVICES" +# Count only non-empty tokens, matching gpu_pool.get_available_gpus (e.g. ",0,1" is 2 GPUs). +_USABLE_GPU_COUNT=0 +for _gpu in "${_VISIBLE_GPUS[@]}"; do + [[ -n "${_gpu//[[:space:]]/}" ]] && _USABLE_GPU_COUNT=$((_USABLE_GPU_COUNT + 1)) +done +if [[ "$FASTVIDEO_GPU_COUNT" =~ ^[0-9]+$ ]] && (( FASTVIDEO_GPU_COUNT < _USABLE_GPU_COUNT )); then + _USABLE_GPU_COUNT="$FASTVIDEO_GPU_COUNT" +fi +export DREAMVERSE_SP_SIZE="${DREAMVERSE_SP_SIZE:-$_USABLE_GPU_COUNT}" PORT="${DREAMVERSE_PORT:-8009}" FFMPEG_ENV="$(dirname "$0")/ffmpeg-env.sh" diff --git a/apps/dreamverse/scripts/setup-dreamverse-env.sh b/apps/dreamverse/scripts/setup-dreamverse-env.sh index 4ab0fd562..15bab7462 100755 --- a/apps/dreamverse/scripts/setup-dreamverse-env.sh +++ b/apps/dreamverse/scripts/setup-dreamverse-env.sh @@ -101,5 +101,5 @@ UV_CACHE_DIR="$LOCAL_DIR/cache" "$CONDA_PREFIX/bin/uv" pip install \ https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.11.post3/flashinfer_python-0.6.11.post3-py3-none-any.whl echo "" -echo "Done. On a compute node run:" -echo " CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh" +echo "Done. On a compute node run (GPUs 0-3 by default; set CUDA_VISIBLE_DEVICES to restrict):" +echo " bash apps/dreamverse/scripts/launch-dreamverse.sh" diff --git a/apps/dreamverse/serve_configs/streaming_demo.yaml b/apps/dreamverse/serve_configs/streaming_demo.yaml index 9db217df2..c4bea239b 100644 --- a/apps/dreamverse/serve_configs/streaming_demo.yaml +++ b/apps/dreamverse/serve_configs/streaming_demo.yaml @@ -110,7 +110,7 @@ default_request: fps: 24 # internal: gpu_pool.py:85 TARGET_FPS streaming: - # internal: config.py:33 SESSION_TIMEOUT_SECONDS = 300 + # internal: config.py SESSION_TIMEOUT_SECONDS (env DREAMVERSE_SESSION_TIMEOUT_SECONDS, default 300) session_timeout_seconds: 300 # internal: config.py:282-284 GENERATION_SEGMENT_CAP default 6 generation_segment_cap: 6