[fix] dreamverse: env-tunable session timeout, restore warmup watchdog, derive SP size in launch script
- SESSION_TIMEOUT_SECONDS: default back to 300 and now reads the DREAMVERSE_SESSION_TIMEOUT_SECONDS env var that launch-dreamverse.sh was already exporting (previously nothing read it, and the hardcoded 1800 made every deployment hold idle GPU-pool slots 6x longer). Fixes the stale five-minute-timeout test and adds an override test. - STARTUP_WARMUP_TIMEOUT_SECONDS: default back to 2400 so the warmup watchdog works again; launch-dreamverse.sh exports the existing FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS override (24000) for the slow GB200 max-autotune boot. - launch-dreamverse.sh: derive the DREAMVERSE_SP_SIZE default from the number of visible GPUs so the documented CUDA_VISIBLE_DEVICES=0 invocation no longer crashes gpu_pool with 'Not enough GPUs'; explicit env still wins. setup-dreamverse-env.sh's printed instruction now matches. gpu_pool raises a friendly error for non-integer CUDA_VISIBLE_DEVICES entries. - video_generation.py: drop unused ParallelismConfig import (F401).
This commit is contained in:
@@ -76,9 +76,6 @@ if ACTIVE_MODEL_ID not in MODEL_REGISTRY:
|
||||
# Active model configuration
|
||||
MODEL_CONFIG = MODEL_REGISTRY[ACTIVE_MODEL_ID]
|
||||
|
||||
# Generation limits
|
||||
SESSION_TIMEOUT_SECONDS = 1800
|
||||
|
||||
# Frame settings
|
||||
NUM_FRAMES = 121
|
||||
FRAME_HEIGHT = 1088
|
||||
@@ -168,6 +165,9 @@ def _optional_env(*names: str) -> str | None:
|
||||
return None
|
||||
|
||||
|
||||
# Generation limits
|
||||
SESSION_TIMEOUT_SECONDS = _env_int("DREAMVERSE_SESSION_TIMEOUT_SECONDS", 300)
|
||||
|
||||
DEVTOOLS_ENABLED = _env_bool("FASTVIDEO_ENABLE_DEVTOOLS", False)
|
||||
PROMPT_SAFETY_ENABLED = _env_bool("FASTVIDEO_ENABLE_PROMPT_SAFETY", False)
|
||||
DREAMVERSE_MAX_AUTOTUNE = _env_bool("DREAMVERSE_MAX_AUTOTUNE", True)
|
||||
@@ -376,4 +376,4 @@ STARTUP_WARMUP_PROMPT = os.getenv(
|
||||
("A cinematic drone shot over coastal cliffs at sunrise, "
|
||||
"golden light, gentle ocean waves, ultra detailed"),
|
||||
).strip()
|
||||
STARTUP_WARMUP_TIMEOUT_SECONDS = max(1, _env_int("FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS", 24000))
|
||||
STARTUP_WARMUP_TIMEOUT_SECONDS = max(1, _env_int("FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS", 2400))
|
||||
|
||||
@@ -1023,7 +1023,11 @@ def get_available_gpus() -> list[int]:
|
||||
"""Get list of available GPU IDs from environment or auto-detect."""
|
||||
cuda_visible = os.environ.get("CUDA_VISIBLE_DEVICES", "")
|
||||
if cuda_visible:
|
||||
visible_gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()]
|
||||
try:
|
||||
visible_gpu_ids = [int(x.strip()) for x in cuda_visible.split(",") if x.strip()]
|
||||
except ValueError as exc:
|
||||
raise RuntimeError("CUDA_VISIBLE_DEVICES must be a comma-separated list of integer GPU "
|
||||
f"indices (got {cuda_visible!r}); GPU UUIDs are not supported.") from exc
|
||||
return _limit_gpu_ids(visible_gpu_ids)
|
||||
|
||||
# Auto-detect available GPUs
|
||||
|
||||
@@ -150,12 +150,22 @@ def test_config_enables_prompt_safety_when_requested(monkeypatch):
|
||||
|
||||
def test_config_uses_five_minute_session_timeout(monkeypatch):
|
||||
_set_required_prompt_keys(monkeypatch)
|
||||
monkeypatch.delenv("DREAMVERSE_SESSION_TIMEOUT_SECONDS", raising=False)
|
||||
|
||||
module = _load_config_module()
|
||||
|
||||
assert module.SESSION_TIMEOUT_SECONDS == 300
|
||||
|
||||
|
||||
def test_config_session_timeout_env_override(monkeypatch):
|
||||
_set_required_prompt_keys(monkeypatch)
|
||||
monkeypatch.setenv("DREAMVERSE_SESSION_TIMEOUT_SECONDS", "1800")
|
||||
|
||||
module = _load_config_module()
|
||||
|
||||
assert module.SESSION_TIMEOUT_SECONDS == 1800
|
||||
|
||||
|
||||
def test_config_rejects_invalid_prompt_provider(monkeypatch):
|
||||
monkeypatch.setenv("FASTVIDEO_PROMPT_PROVIDER", "unsupported")
|
||||
_set_required_prompt_keys(monkeypatch)
|
||||
|
||||
@@ -265,7 +265,6 @@ class VideoGenerationWorker:
|
||||
EngineConfig,
|
||||
GeneratorConfig,
|
||||
OffloadConfig,
|
||||
ParallelismConfig,
|
||||
PipelineSelection,
|
||||
QuantizationConfig,
|
||||
)
|
||||
|
||||
@@ -2,7 +2,8 @@
|
||||
# launch-dreamverse.sh — launch dreamverse-server on a compute node.
|
||||
#
|
||||
# Usage (from repo root):
|
||||
# CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh
|
||||
# bash apps/dreamverse/scripts/launch-dreamverse.sh # GPUs 0-3, SP_SIZE=4
|
||||
# CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh # single GPU, SP_SIZE=1
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
@@ -18,7 +19,9 @@ export DREAMVERSE_MAX_AUTOTUNE=true
|
||||
export LTX2_USE_DISTILLED_SIGMAS=0
|
||||
export LTX2_VIDEO_CONDITIONING_NUM_FRAMES=1
|
||||
export AUDIO_CONDITIONING_NUM_FRAMES=41
|
||||
export DREAMVERSE_SESSION_TIMEOUT_SECONDS=1800
|
||||
export DREAMVERSE_SESSION_TIMEOUT_SECONDS="${DREAMVERSE_SESSION_TIMEOUT_SECONDS:-1800}"
|
||||
# GB200 max-autotune warmup compiles can run for hours; keep the watchdog generous here
|
||||
export FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS="${FASTVIDEO_STARTUP_WARMUP_TIMEOUT_SECONDS:-24000}"
|
||||
export CEREBRAS_API_KEY="${CEREBRAS_API_KEY:-}" # set this in your env or ~/.env
|
||||
export FASTVIDEO_PROMPT_CEREBRAS_MODEL="gpt-oss-120b"
|
||||
export TORCHINDUCTOR_CACHE_DIR="$HOME/.cache/torchinductor"
|
||||
@@ -40,7 +43,19 @@ export LD_LIBRARY_PATH="$CUDA_RT_DIR"
|
||||
|
||||
CUDA_VISIBLE_DEVICES="${CUDA_VISIBLE_DEVICES:-0,1,2,3}"
|
||||
export FASTVIDEO_GPU_COUNT="${FASTVIDEO_GPU_COUNT:-all}"
|
||||
export DREAMVERSE_SP_SIZE="${DREAMVERSE_SP_SIZE:-4}"
|
||||
# Default SP size to the usable GPU count so single-GPU invocations work.
|
||||
# A numeric FASTVIDEO_GPU_COUNT caps the pool below the visible count, and an
|
||||
# SP size above the pool size fails GPUPool startup with "Not enough GPUs".
|
||||
IFS=',' read -ra _VISIBLE_GPUS <<< "$CUDA_VISIBLE_DEVICES"
|
||||
# Count only non-empty tokens, matching gpu_pool.get_available_gpus (e.g. ",0,1" is 2 GPUs).
|
||||
_USABLE_GPU_COUNT=0
|
||||
for _gpu in "${_VISIBLE_GPUS[@]}"; do
|
||||
[[ -n "${_gpu//[[:space:]]/}" ]] && _USABLE_GPU_COUNT=$((_USABLE_GPU_COUNT + 1))
|
||||
done
|
||||
if [[ "$FASTVIDEO_GPU_COUNT" =~ ^[0-9]+$ ]] && (( FASTVIDEO_GPU_COUNT < _USABLE_GPU_COUNT )); then
|
||||
_USABLE_GPU_COUNT="$FASTVIDEO_GPU_COUNT"
|
||||
fi
|
||||
export DREAMVERSE_SP_SIZE="${DREAMVERSE_SP_SIZE:-$_USABLE_GPU_COUNT}"
|
||||
PORT="${DREAMVERSE_PORT:-8009}"
|
||||
|
||||
FFMPEG_ENV="$(dirname "$0")/ffmpeg-env.sh"
|
||||
|
||||
@@ -101,5 +101,5 @@ UV_CACHE_DIR="$LOCAL_DIR/cache" "$CONDA_PREFIX/bin/uv" pip install \
|
||||
https://github.com/flashinfer-ai/flashinfer/releases/download/v0.6.11.post3/flashinfer_python-0.6.11.post3-py3-none-any.whl
|
||||
|
||||
echo ""
|
||||
echo "Done. On a compute node run:"
|
||||
echo " CUDA_VISIBLE_DEVICES=0 bash apps/dreamverse/scripts/launch-dreamverse.sh"
|
||||
echo "Done. On a compute node run (GPUs 0-3 by default; set CUDA_VISIBLE_DEVICES to restrict):"
|
||||
echo " bash apps/dreamverse/scripts/launch-dreamverse.sh"
|
||||
|
||||
@@ -110,7 +110,7 @@ default_request:
|
||||
fps: 24 # internal: gpu_pool.py:85 TARGET_FPS
|
||||
|
||||
streaming:
|
||||
# internal: config.py:33 SESSION_TIMEOUT_SECONDS = 300
|
||||
# internal: config.py SESSION_TIMEOUT_SECONDS (env DREAMVERSE_SESSION_TIMEOUT_SECONDS, default 300)
|
||||
session_timeout_seconds: 300
|
||||
# internal: config.py:282-284 GENERATION_SEGMENT_CAP default 6
|
||||
generation_segment_cap: 6
|
||||
|
||||
Reference in New Issue
Block a user