diff --git a/apps/dreamverse/dreamverse/config.py b/apps/dreamverse/dreamverse/config.py index 3c4402d74..1f14aa678 100644 --- a/apps/dreamverse/dreamverse/config.py +++ b/apps/dreamverse/dreamverse/config.py @@ -76,9 +76,6 @@ if ACTIVE_MODEL_ID not in MODEL_REGISTRY: # Active model configuration MODEL_CONFIG = MODEL_REGISTRY[ACTIVE_MODEL_ID] -# Generation limits -SESSION_TIMEOUT_SECONDS = 1800 - # Frame settings NUM_FRAMES = 121 FRAME_HEIGHT = 1088 @@ -168,6 +165,9 @@ def _optional_env(*names: str) -> str | None: return None +# Generation limits +SESSION_TIMEOUT_SECONDS = _env_int("DREAMVERSE_SESSION_TIMEOUT_SECONDS", 300) + DEVTOOLS_ENABLED = _env_bool("FASTVIDEO_ENABLE_DEVTOOLS", False) PROMPT_SAFETY_ENABLED = _env_bool("FASTVIDEO_ENABLE_PROMPT_SAFETY", False) DREAMVERSE_MAX_AUTOTUNE = _env_bool("DREAMVERSE_MAX_AUTOTUNE", True) @@ -376,4 +376,4 @@ STARTUP_WARMUP_PROMPT = os.getenv( ("A cinematic drone shot over coastal cliffs at sunrise, " "golden light, gentle ocean waves, ultra detailed"), ).strip() -STARTUP_WARMUP_TIMEOUT_SECONDS = max(1, _env_int("FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS", 24000)) +STARTUP_WARMUP_TIMEOUT_SECONDS = max(1, _env_int("FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS", 2400)) diff --git a/apps/dreamverse/dreamverse/gpu_pool.py b/apps/dreamverse/dreamverse/gpu_pool.py index 3a3b98e17..2deff88e0 100644 --- a/apps/dreamverse/dreamverse/gpu_pool.py +++ b/apps/dreamverse/dreamverse/gpu_pool.py @@ -1023,7 +1023,11 @@ def get_available_gpus() -> list[int]: """Get list of available GPU IDs from environment or auto-detect.""" cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES", "") if cuda_visible: - visible_gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()] + try: + visible_gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()] + except ValueError as exc: + raise RuntimeError("CUDA_VISIBLE_DEVICES must be a comma-separated list of integer GPU " + f"indices (got {cuda_visible!r}); GPU UUIDs are not supported.") from exc return _limit_gpu_ids(visible_gpu_ids) # Auto-detect available GPUs diff --git a/apps/dreamverse/dreamverse/tests/test_config.py b/apps/dreamverse/dreamverse/tests/test_config.py index 99a7a0893..872b01dcf 100644 --- a/apps/dreamverse/dreamverse/tests/test_config.py +++ b/apps/dreamverse/dreamverse/tests/test_config.py @@ -150,12 +150,22 @@ def test_config_enables_prompt_safety_when_requested(monkeypatch): def test_config_uses_five_minute_session_timeout(monkeypatch): _set_required_prompt_keys(monkeypatch) + monkeypatch.delenv("DREAMVERSE_SESSION_TIMEOUT_SECONDS", raising=False) module = _load_config_module() assert module.SESSION_TIMEOUT_SECONDS == 300 +def test_config_session_timeout_env_override(monkeypatch): + _set_required_prompt_keys(monkeypatch) + monkeypatch.setenv("DREAMVERSE_SESSION_TIMEOUT_SECONDS", "1800") + + module = _load_config_module() + + assert module.SESSION_TIMEOUT_SECONDS == 1800 + + def test_config_rejects_invalid_prompt_provider(monkeypatch): monkeypatch.setenv("FASTVIDEO_PROMPT_PROVIDER", "unsupported") _set_required_prompt_keys(monkeypatch) diff --git a/apps/dreamverse/dreamverse/video_generation.py b/apps/dreamverse/dreamverse/video_generation.py index be7f5eb63..1a24ee990 100644 --- a/apps/dreamverse/dreamverse/video_generation.py +++ b/apps/dreamverse/dreamverse/video_generation.py @@ -265,7 +265,6 @@ class VideoGenerationWorker: EngineConfig, GeneratorConfig, OffloadConfig, - ParallelismConfig, PipelineSelection, QuantizationConfig, ) diff --git a/apps/dreamverse/scripts/launch-dreamverse.sh b/apps/dreamverse/scripts/launch-dreamverse.sh index dc3e4bd47..33e9ceabe 100755 --- a/apps/dreamverse/scripts/launch-dreamverse.sh +++ b/apps/dreamverse/scripts/launch-dreamverse.sh @@ -2,7 +2,8 @@ # launch-dreamverse.sh — launch dreamverse-server on a compute node. # # Usage (from repo root): -# CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh +# bash apps/dreamverse/scripts/launch-dreamverse.sh # GPUs 0-3, SP_SIZE=4 +# CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh # single GPU, SP_SIZE=1 set -euo pipefail @@ -18,7 +19,9 @@ export DREAMVERSE_MAX_AUTOTUNE=true export LTX2_USE_DISTILLED_SIGMAS=0 export LTX2_VIDEO_CONDITIONING_NUM_FRAMES=1 export AUDIO_CONDITIONING_NUM_FRAMES=41 -export DREAMVERSE_SESSION_TIMEOUT_SECONDS=1800 +export DREAMVERSE_SESSION_TIMEOUT_SECONDS="${DREAMVERSE_SESSION_TIMEOUT_SECONDS:-1800}" +# GB200 max-autotune warmup compiles can run for hours; keep the watchdog generous here +export FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS="${FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS:-24000}" export CEREBRAS_API_KEY="${CEREBRAS_API_KEY:-}" # set this in your env or ~/.env export FASTVIDEO_PROMPT_CEREBRAS_MODEL="gpt-oss-120b" export TORCHINDUCTOR_CACHE_DIR="$HOME/.cache/torchinductor" @@ -40,7 +43,19 @@ export LD_LIBRARY_PATH="$CUDA_RT_DIR" CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}" export FASTVIDEO_GPU_COUNT="${FASTVIDEO_GPU_COUNT:-all}" -export DREAMVERSE_SP_SIZE="${DREAMVERSE_SP_SIZE:-4}" +# Default SP size to the usable GPU count so single-GPU invocations work. +# A numeric FASTVIDEO_GPU_COUNT caps the pool below the visible count, and an +# SP size above the pool size fails GPUPool startup with "Not enough GPUs". +IFS=',' read -ra _VISIBLE_GPUS <<< "$CUDA_VISIBLE_DEVICES" +# Count only non-empty tokens, matching gpu_pool.get_available_gpus (e.g. ",0,1" is 2 GPUs). +_USABLE_GPU_COUNT=0 +for _gpu in "${_VISIBLE_GPUS[@]}"; do + [[ -n "${_gpu//[[:space:]]/}" ]] && _USABLE_GPU_COUNT=$((_USABLE_GPU_COUNT + 1)) +done +if [[ "$FASTVIDEO_GPU_COUNT" =~ ^[0-9]+$ ]] && (( FASTVIDEO_GPU_COUNT < _USABLE_GPU_COUNT )); then + _USABLE_GPU_COUNT="$FASTVIDEO_GPU_COUNT" +fi +export DREAMVERSE_SP_SIZE="${DREAMVERSE_SP_SIZE:-$_USABLE_GPU_COUNT}" PORT="${DREAMVERSE_PORT:-8009}" FFMPEG_ENV="$(dirname "$0")/ffmpeg-env.sh" diff --git a/apps/dreamverse/scripts/setup-dreamverse-env.sh b/apps/dreamverse/scripts/setup-dreamverse-env.sh index 4ab0fd562..15bab7462 100755 --- a/apps/dreamverse/scripts/setup-dreamverse-env.sh +++ b/apps/dreamverse/scripts/setup-dreamverse-env.sh @@ -101,5 +101,5 @@ UV_CACHE_DIR="$LOCAL_DIR/cache" "$CONDA_PREFIX/bin/uv" pip install \ https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.11.post3/flashinfer_python-0.6.11.post3-py3-none-any.whl echo "" -echo "Done. On a compute node run:" -echo " CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh" +echo "Done. On a compute node run (GPUs 0-3 by default; set CUDA_VISIBLE_DEVICES to restrict):" +echo " bash apps/dreamverse/scripts/launch-dreamverse.sh" diff --git a/apps/dreamverse/serve_configs/streaming_demo.yaml b/apps/dreamverse/serve_configs/streaming_demo.yaml index 9db217df2..c4bea239b 100644 --- a/apps/dreamverse/serve_configs/streaming_demo.yaml +++ b/apps/dreamverse/serve_configs/streaming_demo.yaml @@ -110,7 +110,7 @@ default_request: fps: 24 # internal: gpu_pool.py:85 TARGET_FPS streaming: - # internal: config.py:33 SESSION_TIMEOUT_SECONDS = 300 + # internal: config.py SESSION_TIMEOUT_SECONDS (env DREAMVERSE_SESSION_TIMEOUT_SECONDS, default 300) session_timeout_seconds: 300 # internal: config.py:282-284 GENERATION_SEGMENT_CAP default 6 generation_segment_cap: 6