Compare commits
28
Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
ca3e6aa714 | ||
|
|
c9e02f666d | ||
|
|
6b039e80ac | ||
|
|
7c2b447264 | ||
|
|
e136c2b7f8 | ||
|
|
fa2d1324c6 | ||
|
|
cdda1b7535 | ||
|
|
089a5a55a0 | ||
|
|
137d9110ca | ||
|
|
d4f144581d | ||
|
|
5f96664abf | ||
|
|
376e445f0e | ||
|
|
2099dd5533 | ||
|
|
0033a9b11d | ||
|
|
91a1c50909 | ||
|
|
3658b120da | ||
|
|
81b73f6b88 | ||
|
|
d4ed61cc7c | ||
|
|
377c79fa7e | ||
|
|
25940a7e86 | ||
|
|
3f45f2da21 | ||
|
|
6212e7262e | ||
|
|
ba371e33ed | ||
|
|
b7d4b37efd | ||
|
|
c59cf79ea4 | ||
|
|
fab9b3159c | ||
|
|
d377f463f1 | ||
|
|
01050277ea |
@@ -26,7 +26,13 @@ and the coordination contract with that bundle.
|
||||
GPU lane. Each step pins a unique key and label, a 90-minute timeout, the
|
||||
trusted `/opt/fastvideo-ci-runner/run-ci` command (`run-unit` is the one
|
||||
compatibility wrapper), step-level internal `TEST_TYPE`, and
|
||||
`queue: "ci-runner"`.
|
||||
`queue: "ci-runner"` plus a `backend` agent tag.
|
||||
- `backend` is `"${FV_SPARK_BACKEND:-slurm}"` for the Spark-capable
|
||||
(single-GPU) lanes and a literal `"slurm"` for ssim, training,
|
||||
distillation, self-forcing, lora-training, training-vsa and performance.
|
||||
A Spark runner sets `FASTVIDEO_CI_BACKEND=spark4`; gate cluster-only tests on
|
||||
that flag, never on memory heuristics. Keep
|
||||
`SPARK_CAPABLE_LANES` in `test_ci_test_collection.py` in sync.
|
||||
- Active CI contains no `pr_test.sh` command, Modal invocation, default queue,
|
||||
Buildkite plugin, `soft_fail`, or job-controlled artifact glob.
|
||||
- The six Fastcheck lanes use `:microscope:` labels. Full-Suite-only lanes use
|
||||
@@ -45,8 +51,8 @@ and the coordination contract with that bundle.
|
||||
- Focused merge builds may pass allowlisted golden-gate and SSIM test basenames.
|
||||
The private host validates the lane plan and basenames before staging them,
|
||||
and the in-container scripts validate them again. Direct `/test ssim`,
|
||||
explicit `/test full`, and the weekly main-branch schedule run the complete
|
||||
SSIM matrix.
|
||||
explicit `/test full`, and the weekly main-branch Full Suite
|
||||
(`ci-scheduled-full-suite.yml`) run the complete SSIM matrix.
|
||||
- The trusted uploader serves exactly three entry pipelines:
|
||||
`pr-fastcheck` for automatic PR builds, `ci` for slash-command/ready-label
|
||||
API builds, and `fastvideo-performance-lane` for the weekly schedule. Keep
|
||||
|
||||
@@ -23,6 +23,8 @@ notify:
|
||||
# dispatcher, and every test payload executes inside the Slinky Slurm tray.
|
||||
# fastvideo/tests/modal remains available only for an explicit manual rollback;
|
||||
# no active pipeline or slash-command route invokes it.
|
||||
# Agent tag `backend` routes a lane: Spark-capable lanes default to slurm and move to a
|
||||
# DGX Spark runner when FV_SPARK_BACKEND=spark4; the rest are pinned to slurm.
|
||||
# Buildkite hands jobs to free agents in the order they appear here. Golden-gate comes first
|
||||
# because every later merge lane waits for it; the fastcheck lanes follow from longest to
|
||||
# shortest measured runtime, so the longest lane never starts last and stretches the build.
|
||||
@@ -47,6 +49,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":microscope: Unit Tests"
|
||||
key: "unit"
|
||||
@@ -70,6 +73,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":microscope: Kernel Tests"
|
||||
key: "kernel-tests"
|
||||
@@ -93,6 +97,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":microscope: DreamVerse App Tests"
|
||||
key: "dreamverse"
|
||||
@@ -116,6 +121,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":microscope: Encoder Tests"
|
||||
key: "encoder"
|
||||
@@ -139,6 +145,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":microscope: VAE Tests"
|
||||
key: "vae"
|
||||
@@ -162,6 +169,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":microscope: Transformer Tests"
|
||||
key: "transformer"
|
||||
@@ -185,6 +193,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":bar_chart: SSIM Tests"
|
||||
key: "ssim"
|
||||
@@ -212,6 +221,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "slurm"
|
||||
|
||||
- label: ":test_tube: LoRA Inference Tests"
|
||||
key: "lora-inference"
|
||||
@@ -234,6 +244,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":test_tube: LoRA Extraction Tests"
|
||||
key: "lora-extraction"
|
||||
@@ -256,6 +267,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":test_tube: Training Tests"
|
||||
key: "training"
|
||||
@@ -280,6 +292,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "slurm"
|
||||
|
||||
- label: ":test_tube: Distillation DMD Tests"
|
||||
key: "distillation"
|
||||
@@ -302,6 +315,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "slurm"
|
||||
|
||||
- label: ":test_tube: Self-Forcing Tests"
|
||||
key: "self-forcing"
|
||||
@@ -324,6 +338,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "slurm"
|
||||
|
||||
- label: ":test_tube: LoRA Training Tests"
|
||||
key: "lora-training"
|
||||
@@ -348,6 +363,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "slurm"
|
||||
|
||||
- label: ":test_tube: Training Tests VSA"
|
||||
key: "training-vsa"
|
||||
@@ -372,6 +388,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "slurm"
|
||||
|
||||
- label: ":test_tube: Inference Tests VMoBA"
|
||||
key: "inference-vmoba"
|
||||
@@ -394,6 +411,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":test_tube: Performance Tests"
|
||||
key: "performance"
|
||||
@@ -416,6 +434,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "slurm"
|
||||
|
||||
- label: ":test_tube: API Server Tests"
|
||||
key: "api-server"
|
||||
@@ -438,6 +457,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":test_tube: Train Framework Tests"
|
||||
key: "train-framework"
|
||||
@@ -460,6 +480,7 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
- label: ":test_tube: Eval Metrics Tests"
|
||||
key: "eval"
|
||||
@@ -482,3 +503,4 @@ steps:
|
||||
limit: 2
|
||||
agents:
|
||||
queue: "ci-runner"
|
||||
backend: "${FV_SPARK_BACKEND:-slurm}"
|
||||
|
||||
@@ -75,7 +75,7 @@ if [[ $machine_arch =~ ^(aarch64|arm64)$ ]]; then
|
||||
npm run e2e -- \
|
||||
--project=chromium \
|
||||
--project=mobile-chromium \
|
||||
--grep-invert='streams, plays, and surfaces a downloadable clip|starts a new project and switches back to the prior session|saved projects persist across a page reload'
|
||||
--grep-invert='streams, plays, and surfaces a downloadable clip|starts a new project and switches back to the prior session|saved projects persist across a page reload|sends validated assets and plays a clearly labeled sample'
|
||||
else
|
||||
BACKEND_HOST=127.0.0.1 BACKEND_PORT="$BACKEND_PORT" CI=1 \
|
||||
npm run e2e -- \
|
||||
|
||||
@@ -2,4 +2,4 @@
|
||||
# Canonical Slurm CI selection for the VMoBA-inference lane.
|
||||
set -euo pipefail
|
||||
|
||||
exec python fastvideo/tests/inference/vmoba/test_vmoba_inference.py
|
||||
exec pytest ./fastvideo/tests/inference/vmoba/test_vmoba_inference.py -vs
|
||||
|
||||
@@ -9,9 +9,12 @@ mkdir -p "$PERF_REPORTS_DIR"
|
||||
|
||||
if [[ ${BUILDKITE_PULL_REQUEST:-false} =~ ^[1-9][0-9]*$ ]]; then
|
||||
export PERF_RUN_SOURCE=pr
|
||||
export PERF_UPLOAD_POLICY=pass
|
||||
export PERF_UPLOAD_POLICY=never
|
||||
elif [ "${BUILDKITE_BRANCH:-}" = main ] \
|
||||
&& { [ "${BUILDKITE_SOURCE:-}" = schedule ] || [ "${TEST_SCOPE:-}" = full ]; }; then
|
||||
&& { [ "${BUILDKITE_SOURCE:-}" = schedule ] || [ "${TEST_SCOPE:-}" = full ] \
|
||||
|| { [ "${BUILDKITE_SOURCE:-}" = api ] \
|
||||
&& [ "${BUILDKITE_PIPELINE_SLUG:-}" = fastvideo-performance-lane ] \
|
||||
&& [ "${TEST_SCOPE:-}" = direct ]; }; }; then
|
||||
export PERF_RUN_SOURCE=scheduled_main
|
||||
export PERF_UPLOAD_POLICY=always
|
||||
elif [ "${TEST_SCOPE:-}" = direct ]; then
|
||||
|
||||
@@ -31,6 +31,7 @@ exec pytest \
|
||||
./fastvideo/tests/inference/test_inference_regional_compile.py \
|
||||
./fastvideo/tests/attention/ \
|
||||
./fastvideo/tests/layers/test_pdd_linear.py \
|
||||
./fastvideo/tests/layers/test_lora_side_branch.py \
|
||||
./fastvideo/tests/layers/test_triton_fused_norm.py \
|
||||
./fastvideo/tests/modal/test_kernel_build_cache.py \
|
||||
./fastvideo/tests/modal/test_pr_test.py \
|
||||
|
||||
@@ -104,9 +104,9 @@ SAFE_PATTERNS = (
|
||||
"comfyui/**",
|
||||
"docs/**",
|
||||
"examples/**",
|
||||
"mkdocs.yml",
|
||||
"requirements-mkdocs.in",
|
||||
"requirements-mkdocs.txt",
|
||||
"docs-site/**",
|
||||
"requirements-docs.in",
|
||||
"requirements-docs.txt",
|
||||
"scripts/**",
|
||||
"tests/__init__.py",
|
||||
"tests/local_tests/**",
|
||||
@@ -321,7 +321,7 @@ def classify_paths(paths: list[str], removed_paths: Iterable[str] = ()) -> Merge
|
||||
plan.require_all("changed-file API failed; failing closed")
|
||||
continue
|
||||
|
||||
if path in {"requirements-mkdocs.in", "requirements-mkdocs.txt"}:
|
||||
if path in {"requirements-docs.in", "requirements-docs.txt"}:
|
||||
plan.reasons.append(f"documentation dependencies need no GPU integration: {path}")
|
||||
continue
|
||||
|
||||
|
||||
@@ -88,6 +88,13 @@ jobs:
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_vsa_regressions.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_fast_mode.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_fast_spatial.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_ref_layout.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_pdd.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_reference_prep.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_ref2va_pipeline.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_ref_conditioning.py \
|
||||
fastvideo/tests/mlx/test_mlx_qwen3_vl_vision.py \
|
||||
fastvideo/tests/mlx/test_mlx_fasth3_omniref_example.py \
|
||||
fastvideo/tests/mlx/test_mlx_fastwan_benchmark.py \
|
||||
fastvideo/tests/mlx/test_taehv_decode.py \
|
||||
fastvideo/tests/mlx/test_frame_upsample.py \
|
||||
@@ -153,6 +160,13 @@ jobs:
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_vsa_regressions.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_fast_mode.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_fast_spatial.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_ref_layout.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_pdd.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_reference_prep.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_ref2va_pipeline.py \
|
||||
fastvideo/tests/mlx/test_mlx_minimax_h3_ref_conditioning.py \
|
||||
fastvideo/tests/mlx/test_mlx_qwen3_vl_vision.py \
|
||||
fastvideo/tests/mlx/test_mlx_fasth3_omniref_example.py \
|
||||
fastvideo/tests/mlx/test_mlx_fastwan_benchmark.py \
|
||||
fastvideo/tests/mlx/test_taehv_decode.py \
|
||||
fastvideo/tests/mlx/test_frame_upsample.py \
|
||||
|
||||
+7
-7
@@ -1,4 +1,4 @@
|
||||
name: Scheduled Full SSIM
|
||||
name: Scheduled Full Suite
|
||||
|
||||
on:
|
||||
schedule:
|
||||
@@ -13,7 +13,7 @@ jobs:
|
||||
if: github.repository == 'hao-ai-lab/FastVideo'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Trigger weekly full SSIM on Slinky Slurm
|
||||
- name: Trigger weekly full suite on Slinky Slurm
|
||||
env:
|
||||
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
|
||||
SOURCE_SHA: ${{ github.sha }}
|
||||
@@ -32,13 +32,13 @@ jobs:
|
||||
'{
|
||||
commit: $commit,
|
||||
branch: $branch,
|
||||
message: "Weekly full SSIM on Slinky Slurm",
|
||||
message: "Weekly full suite on main",
|
||||
ignore_pipeline_branch_filters: true,
|
||||
env: {
|
||||
TEST_SCOPE: "scheduled",
|
||||
FULL_SUITE: "false",
|
||||
TEST_TYPE: "ssim",
|
||||
TEST_SCOPE: "full",
|
||||
FULL_SUITE: "true",
|
||||
TEST_TYPE: "all",
|
||||
PR_NUMBER: "false",
|
||||
PR_TITLE: "Scheduled full SSIM"
|
||||
PR_TITLE: "Scheduled full suite"
|
||||
}
|
||||
}')"
|
||||
@@ -0,0 +1,73 @@
|
||||
name: Scheduled Performance
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Sunday 23:00 America/Los_Angeles, including daylight saving time.
|
||||
- cron: "0 6 * * 1"
|
||||
- cron: "0 7 * * 1"
|
||||
workflow_dispatch:
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
trigger:
|
||||
if: github.repository == 'hao-ai-lab/FastVideo' && github.ref == 'refs/heads/main'
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Select the Pacific-time schedule
|
||||
id: schedule
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
SCHEDULE_EXPR: ${{ github.event.schedule }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ "$EVENT_NAME" == schedule ]]; then
|
||||
expected='0 7 * * 1'
|
||||
if [[ "$(TZ=America/Los_Angeles date +%z)" == -0700 ]]; then
|
||||
expected='0 6 * * 1'
|
||||
fi
|
||||
if [[ "$SCHEDULE_EXPR" != "$expected" ]]; then
|
||||
echo 'run=false' >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
fi
|
||||
echo 'run=true' >> "$GITHUB_OUTPUT"
|
||||
- name: Trigger weekly performance on Slinky Slurm
|
||||
if: steps.schedule.outputs.run == 'true'
|
||||
env:
|
||||
BUILDKITE_API_TOKEN: ${{ secrets.BUILDKITE_API_TOKEN }}
|
||||
SOURCE_SHA: ${{ github.sha }}
|
||||
SOURCE_BRANCH: ${{ github.event.repository.default_branch }}
|
||||
BK_ORG: ${{ vars.BUILDKITE_ORG_SLUG }}
|
||||
BK_PIPELINE: fastvideo-performance-lane
|
||||
run: |
|
||||
set -euo pipefail
|
||||
if [[ ${#SOURCE_SHA} -ne 40 || "$SOURCE_SHA" == *[!0-9a-f]* ]]; then
|
||||
echo 'SOURCE_SHA must be a full immutable commit SHA' >&2
|
||||
exit 1
|
||||
fi
|
||||
payload=$(python3 - <<'PY'
|
||||
import json
|
||||
import os
|
||||
|
||||
print(json.dumps({
|
||||
"commit": os.environ["SOURCE_SHA"],
|
||||
"branch": os.environ["SOURCE_BRANCH"],
|
||||
"message": "Weekly performance on Slinky Slurm",
|
||||
"ignore_pipeline_branch_filters": True,
|
||||
"env": {
|
||||
"TEST_SCOPE": "direct",
|
||||
"FULL_SUITE": "false",
|
||||
"TEST_TYPE": "performance",
|
||||
"PR_NUMBER": "false",
|
||||
"PR_TITLE": "Scheduled performance",
|
||||
},
|
||||
}))
|
||||
PY
|
||||
)
|
||||
curl -sS --fail-with-body --connect-timeout 10 --max-time 60 -X POST \
|
||||
"https://api.buildkite.com/v2/organizations/${BK_ORG}/pipelines/${BK_PIPELINE}/builds" \
|
||||
-H "Authorization: Bearer $BUILDKITE_API_TOKEN" \
|
||||
-H "Content-Type: application/json" \
|
||||
--data-raw "$payload"
|
||||
@@ -1,34 +0,0 @@
|
||||
name: Close Stale Issues and PRs
|
||||
|
||||
on:
|
||||
schedule:
|
||||
# Daily at 1:30 AM UTC
|
||||
- cron: '30 1 * * *'
|
||||
|
||||
jobs:
|
||||
stale:
|
||||
if: github.repository == 'hao-ai-lab/FastVideo'
|
||||
permissions:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
actions: write
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/stale@997185467fa4f803885201cee163a9f38240193d # v10.1.1
|
||||
with:
|
||||
operations-per-run: 500
|
||||
|
||||
exempt-draft-pr: true
|
||||
exempt-issue-labels: 'keep-open,pinned,security,Bug,RFC'
|
||||
exempt-pr-labels: 'keep-open,pinned'
|
||||
|
||||
labels-to-add-when-unstale: 'unstale'
|
||||
labels-to-remove-when-stale: 'unstale'
|
||||
|
||||
days-before-issue-stale: 90
|
||||
days-before-issue-close: 30
|
||||
stale-issue-label: 'stale'
|
||||
|
||||
days-before-pr-stale: 60
|
||||
days-before-pr-close: 14
|
||||
stale-pr-label: 'stale'
|
||||
@@ -56,7 +56,9 @@ jobs:
|
||||
# CUDA matrix: Python 3.12 x {12.6.3, 13.0.0} x {amd64, arm64}. Each architecture
|
||||
# builds natively and pushes only by digest; publish-cuda-manifests is the sole
|
||||
# owner of the shared tags. CUDA 12.6 arm64 targets Hopper (sm_90a / GH200 class),
|
||||
# because CUDA 12.6 cannot compile sm_121. CUDA 13 arm64 targets GB10 (sm_121).
|
||||
# because CUDA 12.6 cannot compile sm_121. CUDA 13 arm64 targets GB10 as 12.1a
|
||||
# (sm_121a), as a native Spark build does: the plain 12.1 target drops the
|
||||
# arch-specific attn_qat_infer FP4 kernels.
|
||||
# 12.6.3/cu126 owns the default `py3.12`/`latest` tags and keeps its versioned
|
||||
# aliases; 13.0.0/cu130 is published under explicit versioned tags. Flash-attn
|
||||
# 2.8.3 comes from the architecture-specific prebuilt releases.
|
||||
@@ -78,6 +80,12 @@ jobs:
|
||||
torch_cuda_arch_list:
|
||||
amd64: '9.0a'
|
||||
arm64: '9.0a'
|
||||
install_flash_attn:
|
||||
amd64: '1'
|
||||
arm64: '1'
|
||||
install_fa4_cute:
|
||||
amd64: '1'
|
||||
arm64: '1'
|
||||
- version: '13.0.0'
|
||||
suffix: '-cuda13.0.0'
|
||||
torch_backend: 'cu130'
|
||||
@@ -85,7 +93,17 @@ jobs:
|
||||
cmake_build_parallel_level: '1'
|
||||
torch_cuda_arch_list:
|
||||
amd64: '9.0a'
|
||||
arm64: '12.1'
|
||||
arm64: '12.1a'
|
||||
# FA4 cute does not run on GB10 (sm_12x), and its cutlass-dsl pin
|
||||
# breaks flashinfer there; GB10 FP4 attention uses fastvideo-kernel.
|
||||
# FA2 gives no measured GB10 speedup and changes the output relative
|
||||
# to a native Spark install, so the GB10 image matches that install.
|
||||
install_flash_attn:
|
||||
amd64: '1'
|
||||
arm64: '0'
|
||||
install_fa4_cute:
|
||||
amd64: '1'
|
||||
arm64: '0'
|
||||
architecture:
|
||||
- name: amd64
|
||||
runner: ubuntu-latest
|
||||
@@ -105,6 +123,8 @@ jobs:
|
||||
CUDA_VERSION=${{ matrix.cuda.version }}
|
||||
UV_TORCH_BACKEND=${{ matrix.cuda.torch_backend }}
|
||||
TORCH_CUDA_ARCH_LIST=${{ matrix.cuda.torch_cuda_arch_list[matrix.architecture.name] }}
|
||||
INSTALL_FLASH_ATTN=${{ matrix.cuda.install_flash_attn[matrix.architecture.name] }}
|
||||
INSTALL_FA4_CUTE=${{ matrix.cuda.install_fa4_cute[matrix.architecture.name] }}
|
||||
CMAKE_BUILD_PARALLEL_LEVEL=${{ matrix.cuda.cmake_build_parallel_level }}
|
||||
FLASH_ATTN_WHEEL_TAG=${{ matrix.cuda.fa_tag }}
|
||||
FLASH_ATTN_WHEEL_RELEASE=https://github.com/mjun0812/flash-attention-prebuild-wheels/releases/download/v0.9.17
|
||||
|
||||
@@ -7,9 +7,11 @@ on:
|
||||
- 'docs/**'
|
||||
- 'examples/**'
|
||||
- 'scripts/inference/**'
|
||||
- 'mkdocs.yml'
|
||||
- 'requirements-mkdocs.in'
|
||||
- 'requirements-mkdocs.txt'
|
||||
- 'docs-site/**'
|
||||
- 'fastvideo/**'
|
||||
- 'README.md'
|
||||
- 'requirements-docs.in'
|
||||
- 'requirements-docs.txt'
|
||||
- 'scripts/check_docs_links.py'
|
||||
- '.github/workflows/infra-docs.yml'
|
||||
pull_request:
|
||||
@@ -18,9 +20,11 @@ on:
|
||||
- 'docs/**'
|
||||
- 'examples/**'
|
||||
- 'scripts/inference/**'
|
||||
- 'mkdocs.yml'
|
||||
- 'requirements-mkdocs.in'
|
||||
- 'requirements-mkdocs.txt'
|
||||
- 'docs-site/**'
|
||||
- 'fastvideo/**'
|
||||
- 'README.md'
|
||||
- 'requirements-docs.in'
|
||||
- 'requirements-docs.txt'
|
||||
- 'scripts/check_docs_links.py'
|
||||
- '.github/workflows/infra-docs.yml'
|
||||
|
||||
@@ -35,6 +39,7 @@ concurrency:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
timeout-minutes: 15
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Checkout
|
||||
@@ -51,7 +56,7 @@ jobs:
|
||||
uses: astral-sh/setup-uv@v3
|
||||
|
||||
- name: Install dependencies
|
||||
run: uv pip install --system -r requirements-mkdocs.txt
|
||||
run: uv pip install --system -r requirements-docs.txt
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v4
|
||||
@@ -59,10 +64,16 @@ jobs:
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
node-version: '24'
|
||||
cache: npm
|
||||
cache-dependency-path: |
|
||||
docs/package-lock.json
|
||||
docs-site/package-lock.json
|
||||
|
||||
- name: Install cookbook dependencies
|
||||
run: npm ci --prefix docs
|
||||
run: |
|
||||
npm ci --prefix docs
|
||||
npm ci --prefix docs-site
|
||||
|
||||
- name: Validate and generate cookbook catalogs
|
||||
run: |
|
||||
@@ -71,8 +82,25 @@ jobs:
|
||||
python docs/tests/test_cookbook_assets.py
|
||||
npm run build:catalog --prefix docs
|
||||
|
||||
- name: Build documentation
|
||||
run: mkdocs build
|
||||
- name: Check site types and formatting
|
||||
run: |
|
||||
npm run check --prefix docs-site
|
||||
npm run format:check --prefix docs-site
|
||||
|
||||
- name: Build and validate documentation
|
||||
run: |
|
||||
npm run build --prefix docs-site
|
||||
npm run validate --prefix docs-site
|
||||
npm run test:content --prefix docs-site
|
||||
python -m unittest discover -s docs-site/tests -p 'test_*.py'
|
||||
python docs/check_cookbook_assets.py docs-site/dist
|
||||
|
||||
- name: Install test browser
|
||||
working-directory: docs-site
|
||||
run: npx playwright install --with-deps chromium
|
||||
|
||||
- name: Test documentation and cookbook interactions
|
||||
run: npm test --prefix docs-site
|
||||
|
||||
- name: Check docs links
|
||||
run: python scripts/check_docs_links.py
|
||||
@@ -80,7 +108,15 @@ jobs:
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v3
|
||||
with:
|
||||
path: ./site
|
||||
path: ./docs-site/dist
|
||||
|
||||
- name: Upload browser failure traces
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: docs-browser-results
|
||||
path: docs-site/test-results
|
||||
if-no-files-found: ignore
|
||||
|
||||
deploy:
|
||||
environment:
|
||||
|
||||
+1
-3
@@ -52,9 +52,8 @@ dist/
|
||||
eggs/
|
||||
.eggs/
|
||||
|
||||
# MkDocs documentation
|
||||
# Generated documentation
|
||||
site/
|
||||
docs/assets/cookbook-serving.json
|
||||
docs/assets/cookbook-config/
|
||||
examples/serving/clients/node_modules/
|
||||
docs/node_modules/
|
||||
@@ -63,7 +62,6 @@ docs/examples/
|
||||
docs/inference/examples/
|
||||
docs/training/examples/
|
||||
docs/distillation/examples/
|
||||
!requirements-mkdocs.txt
|
||||
|
||||
# VSCode
|
||||
.vscode/
|
||||
|
||||
@@ -21,6 +21,7 @@ repos:
|
||||
rev: v0.43.0
|
||||
hooks:
|
||||
- id: yapf
|
||||
require_serial: true
|
||||
args: [--in-place, --verbose]
|
||||
language_version: python3.12
|
||||
additional_dependencies: [toml] # TODO: Remove when yapf is upgraded
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
- Tests:
|
||||
- `fastvideo/tests/` for package-level tests (dataset, encoders, inference, training, SSIM, workflow).
|
||||
- `tests/local_tests/` for additional local/component checks.
|
||||
- Docs and guides: `docs/` (MkDocs source), with contributor docs in `docs/contributing/`.
|
||||
- Docs and guides: canonical Markdown in `docs/`, the Blume app in `docs-site/`, and contributor docs in `docs/contributing/`.
|
||||
- Runnable examples and scripts: `examples/` and `scripts/`.
|
||||
- Static assets: `assets/` (including `assets/images/`, `assets/videos/`, and `assets/prompts/`) and `comfyui/assets/`.
|
||||
|
||||
|
||||
@@ -48,7 +48,7 @@ class MiniMaxH3GenerationBackend:
|
||||
"FASTVIDEO_ATTENTION_BACKEND": attention_backend,
|
||||
"FASTVIDEO_FA4": "1",
|
||||
"FASTVIDEO_MINIMAX_H3_FUSIONS": "all",
|
||||
"FASTVIDEO_VSA_SM100A": "0",
|
||||
"FASTVIDEO_VSA_PTX_BLACKWELL": "0",
|
||||
})
|
||||
os.environ.pop("FASTVIDEO_INFERENCE_TORCH_COMPILE", None)
|
||||
|
||||
|
||||
@@ -73,7 +73,7 @@ def test_initialize_builds_vsa_datafree_fasth3_generator(monkeypatch):
|
||||
monkeypatch.setenv("FASTVIDEO_ATTENTION_BACKEND", "test-attention")
|
||||
monkeypatch.setenv("FASTVIDEO_FA4", "0")
|
||||
monkeypatch.setenv("FASTVIDEO_MINIMAX_H3_FUSIONS", "0")
|
||||
monkeypatch.setenv("FASTVIDEO_VSA_SM100A", "1")
|
||||
monkeypatch.setenv("FASTVIDEO_VSA_PTX_BLACKWELL", "1")
|
||||
monkeypatch.setenv("FASTVIDEO_INFERENCE_TORCH_COMPILE", "1")
|
||||
|
||||
backend = MiniMaxH3GenerationBackend(gpu_id=0)
|
||||
@@ -108,7 +108,7 @@ def test_initialize_builds_vsa_datafree_fasth3_generator(monkeypatch):
|
||||
assert os.environ["FASTVIDEO_ATTENTION_BACKEND"] == "VIDEO_SPARSE_ATTN_H3"
|
||||
assert os.environ["FASTVIDEO_FA4"] == "1"
|
||||
assert os.environ["FASTVIDEO_MINIMAX_H3_FUSIONS"] == "all"
|
||||
assert os.environ["FASTVIDEO_VSA_SM100A"] == "0"
|
||||
assert os.environ["FASTVIDEO_VSA_PTX_BLACKWELL"] == "0"
|
||||
assert "FASTVIDEO_INFERENCE_TORCH_COMPILE" not in os.environ
|
||||
|
||||
|
||||
|
||||
@@ -3,6 +3,37 @@ import { stat } from 'node:fs/promises';
|
||||
import { test, expect, type WebSocket as PWWebSocket } from '@playwright/test';
|
||||
|
||||
test.describe('mock-backed generation smoke', () => {
|
||||
test('leave dialog blocks navigation and keeps cancel and confirm usable', async ({ page, request }) => {
|
||||
const health = await request.get('/healthz');
|
||||
const body = health.ok() ? await health.json() : {};
|
||||
test.skip(body.service !== 'ltx2-streaming-mock-server', 'Requires the local mock runtime.');
|
||||
await page.goto('/');
|
||||
await page.getByLabel('Continuation prompt').fill('A paper fox walks through a sunlit forest.');
|
||||
await page.getByRole('button', { name: 'Generate', exact: true }).click();
|
||||
const leave = page.getByRole('button', { name: 'Leave', exact: true });
|
||||
await expect(leave).toBeEnabled();
|
||||
await leave.click();
|
||||
const dialog = page.getByRole('dialog', { name: 'Leave session?' });
|
||||
await expect(dialog).toBeVisible();
|
||||
expect(await dialog.evaluate((element) => element.closest('.isolate') === null)).toBe(true);
|
||||
await expect(page.getByRole('button', { name: 'Toggle sidebar' }).click({ trial: true, timeout: 1000 })).rejects.toThrow();
|
||||
for (let index = 0; index < 4; index++) {
|
||||
await page.keyboard.press('Tab');
|
||||
await expect.poll(() => dialog.evaluate((element) => element.contains(document.activeElement))).toBe(true);
|
||||
}
|
||||
await dialog.getByRole('button', { name: 'Cancel', exact: true }).click();
|
||||
await expect(dialog).toHaveCount(0);
|
||||
await expect(leave).toBeFocused();
|
||||
await leave.click();
|
||||
await page.keyboard.press('Escape');
|
||||
await expect(dialog).toHaveCount(0);
|
||||
await expect(leave).toBeFocused();
|
||||
await leave.click();
|
||||
await dialog.getByRole('button', { name: 'Leave session', exact: true }).click();
|
||||
await expect(dialog).toHaveCount(0);
|
||||
await expect(page.getByRole('button', { name: 'Generate', exact: true })).toBeVisible();
|
||||
});
|
||||
|
||||
test('accepts a custom prompt and enters the streaming state', async ({ page, request }) => {
|
||||
const health = await request.get('/healthz');
|
||||
const body = health.ok() ? await health.json() : {};
|
||||
|
||||
@@ -2739,7 +2739,7 @@ export default function Page() {
|
||||
<Header timeLeft={headerTimeLeft} formatTime={formatTime} onToggleSidebar={() => setSidebarOpen((prev) => !prev)} />
|
||||
|
||||
<div className={cn(
|
||||
"relative flex flex-1 min-h-0 flex-col px-4 pb-2 sm:px-6 sm:pb-12",
|
||||
"relative isolate flex flex-1 min-h-0 flex-col px-4 pb-2 sm:px-6 sm:pb-12",
|
||||
!isViewingMode && !showActiveProject && generationMode !== "t2va"
|
||||
? "justify-start overflow-y-auto pt-4"
|
||||
: "justify-center",
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"use client";
|
||||
|
||||
import { useState, useEffect } from "react";
|
||||
import { useState, useEffect, useRef } from "react";
|
||||
import { Dialog as DialogPrimitive } from "radix-ui";
|
||||
import { Button } from "@/components/ui/button";
|
||||
import { Checkbox } from "@/components/ui/checkbox";
|
||||
import { Label } from "@/components/ui/label";
|
||||
@@ -19,6 +20,7 @@ export default function LeaveSessionModal({
|
||||
onConfirmLeave = () => {},
|
||||
}: LeaveSessionModalProps) {
|
||||
const [suppress, setSuppress] = useState(false);
|
||||
const previousFocus = useRef<HTMLElement | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
if (open) setSuppress(false);
|
||||
@@ -36,43 +38,58 @@ export default function LeaveSessionModal({
|
||||
}
|
||||
|
||||
return (
|
||||
<div className="fixed inset-0 z-[70] flex items-center justify-center bg-black/45 px-4 backdrop-blur-[3px]">
|
||||
<div
|
||||
role="dialog"
|
||||
aria-modal="true"
|
||||
aria-labelledby="leave-session-title"
|
||||
className="w-full max-w-md rounded-3xl border border-border/70 bg-card/95 p-6 shadow-2xl"
|
||||
>
|
||||
<div className="space-y-3">
|
||||
<div className="space-y-1">
|
||||
<h2 id="leave-session-title" className="text-lg font-semibold text-foreground">
|
||||
Leave session?
|
||||
</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
This will end your current session. You will need to queue again for a GPU to start a new project.
|
||||
</p>
|
||||
<DialogPrimitive.Root open={open} onOpenChange={(nextOpen) => { if (!nextOpen) onClose(); }}>
|
||||
<DialogPrimitive.Portal>
|
||||
<DialogPrimitive.Overlay asChild>
|
||||
<div className="fixed inset-0 z-[70] flex items-center justify-center bg-black/45 px-4 backdrop-blur-[3px]">
|
||||
<DialogPrimitive.Content
|
||||
asChild
|
||||
onInteractOutside={(event) => event.preventDefault()}
|
||||
onOpenAutoFocus={() => {
|
||||
previousFocus.current = document.activeElement instanceof HTMLElement ? document.activeElement : null;
|
||||
}}
|
||||
onCloseAutoFocus={(event) => {
|
||||
event.preventDefault();
|
||||
if (previousFocus.current?.isConnected) previousFocus.current.focus();
|
||||
}}
|
||||
>
|
||||
<div
|
||||
className="w-full max-w-md rounded-3xl border border-border/70 bg-card/95 p-6 shadow-2xl"
|
||||
>
|
||||
<div className="space-y-3">
|
||||
<div className="space-y-1">
|
||||
<DialogPrimitive.Title className="text-lg font-semibold text-foreground">
|
||||
Leave session?
|
||||
</DialogPrimitive.Title>
|
||||
<DialogPrimitive.Description className="text-sm text-muted-foreground">
|
||||
This will end your current session. You will need to queue again for a GPU to start a new project.
|
||||
</DialogPrimitive.Description>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Checkbox
|
||||
id="suppress-leave-warning"
|
||||
checked={suppress}
|
||||
onCheckedChange={(v) => setSuppress(v === true)}
|
||||
/>
|
||||
<Label htmlFor="suppress-leave-warning" className="text-sm text-muted-foreground cursor-pointer">
|
||||
Do not warn again
|
||||
</Label>
|
||||
</div>
|
||||
</div>
|
||||
<div className="mt-5 flex flex-col-reverse gap-2 sm:flex-row sm:justify-end">
|
||||
<Button variant="outline" onClick={onClose}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button variant="destructive" onClick={handleConfirm}>
|
||||
Leave session
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</DialogPrimitive.Content>
|
||||
</div>
|
||||
<div className="flex items-center gap-2">
|
||||
<Checkbox
|
||||
id="suppress-leave-warning"
|
||||
checked={suppress}
|
||||
onCheckedChange={(v) => setSuppress(v === true)}
|
||||
/>
|
||||
<Label htmlFor="suppress-leave-warning" className="text-sm text-muted-foreground cursor-pointer">
|
||||
Do not warn again
|
||||
</Label>
|
||||
</div>
|
||||
</div>
|
||||
<div className="mt-5 flex flex-col-reverse gap-2 sm:flex-row sm:justify-end">
|
||||
<Button variant="outline" onClick={onClose}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button variant="destructive" onClick={handleConfirm}>
|
||||
Leave session
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</DialogPrimitive.Overlay>
|
||||
</DialogPrimitive.Portal>
|
||||
</DialogPrimitive.Root>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -16,6 +16,9 @@ The UI currently supports:
|
||||
```
|
||||
apps/fastvideo_studio/
|
||||
├── server.py / job_runner.py / database.py # FastAPI backend + job lifecycle
|
||||
├── job_queue.py # Queue scheduling rules + last-frame references (GPU-free)
|
||||
├── merge.py # Join a scene's clips into one video (ffmpeg stream copy, GPU-free)
|
||||
├── trim.py # Cut a clip to an in/out range, non-destructively (GPU-free)
|
||||
├── mock_server.py # In-memory API mock for e2e tests
|
||||
├── models/ # Pydantic request models (shared with the mock)
|
||||
├── training_config.py # Studio workloads → fastvideo/train YAML configs
|
||||
@@ -88,6 +91,93 @@ Running it this way allows you to pass command line parameters.
|
||||
The API server defaults to port 8189. To configure this, you can edit `.env.local` file.
|
||||
Refer to `.env.example` for reference.
|
||||
|
||||
### Job queue
|
||||
|
||||
`POST /api/jobs/queue` puts jobs in a queue; the server runs them oldest-first,
|
||||
`--max-concurrent-jobs` at a time (default 1). Start on a job while that many
|
||||
are already running is refused; queue it instead. Queued jobs survive a server
|
||||
restart and carry on.
|
||||
|
||||
Inference jobs run on that many long-lived worker threads. The model's GPU
|
||||
processes are killed when the thread that started them exits, so a worker that
|
||||
never exits is what keeps the model loaded from one job to the next (it loads
|
||||
once, not per job). A worker holds one model at a time: a job with a different
|
||||
model or settings replaces it, and a failed or stopped job unloads it so the next
|
||||
one starts clean. Each extra worker loads its own copy of the model.
|
||||
|
||||
A job can start from how another job's video ended, two ways:
|
||||
|
||||
* **Last frame** -- a reference whose source is `job-last-frame:<job id>`. A
|
||||
hard, exact opening frame: the prompt is rewritten to say the shot begins
|
||||
from it.
|
||||
* **Last clip** -- a reference whose source is `job-last-clip:<job id>`, and
|
||||
`media_type: "video"`. The trailing ~1s of the source job's video instead of
|
||||
a single still, so the model has real motion to continue rather than a pose
|
||||
to guess it from. Costs one of H3's 3 video-reference slots instead of one
|
||||
of its 9 image slots.
|
||||
|
||||
Either is extracted when the job runs, so the reference can be added before
|
||||
that job has finished, and the queue holds the job until it has. If that job
|
||||
fails, is stopped, or is deleted, the waiting job fails with a message saying
|
||||
so (and so does anything waiting on *it*); fix the earlier job and queue them
|
||||
again. Starting such a job directly while its source is unfinished is refused
|
||||
with a 409.
|
||||
|
||||
### Merging a scene
|
||||
|
||||
**Merge scene** on the Scenes page joins a scene's clips, in the order shown (the
|
||||
take chosen for each), into one video under `<output-dir>/merged/`, then plays it
|
||||
and offers a download. It is enabled once every clip has finished, and refuses
|
||||
otherwise, so a scene missing a clip can't pass for the whole thing.
|
||||
|
||||
The clips are copied, not re-encoded (about a second for 15 clips, no quality
|
||||
loss). That needs them to share size, frame rate and audio, which clips from one
|
||||
scene do; if one differs, the merge says which. Each clip is cut at the end of its
|
||||
last video frame, because H3's audio runs a few milliseconds longer and would
|
||||
otherwise leave a small gap at every join. It uses `ffmpeg` from `PATH`,
|
||||
`$FASTVIDEO_FFMPEG_BIN`, or the copy bundled with `imageio-ffmpeg`.
|
||||
|
||||
### Editing a clip's video
|
||||
|
||||
**Edit video** on a finished clip in Scenes cuts it to a chosen in/out range and
|
||||
adjusts its brightness, contrast and saturation, in place -- other jobs'
|
||||
last-frame/last-clip references, and a later merge, all just read the job's
|
||||
`output_path`, so nothing else needs to know a clip was edited. Frame-accurate
|
||||
(re-encoded, not stream-copied), so the cut isn't limited to the source's
|
||||
keyframes. Audio is cut to the same range and carried over too (re-encoded to
|
||||
AAC regardless of the source's codec), unless the source has none.
|
||||
|
||||
Color grading is per **section**, not the whole clip at once, like a razor/blade
|
||||
tool: **Split at current time** cuts the kept range wherever the preview is
|
||||
paused, and each resulting section gets its own brightness, contrast and
|
||||
saturation sliders, next to a before/after preview of a still frame
|
||||
(**Preview** on a section seeks to its start and captures that frame; **Use
|
||||
current frame** captures wherever the preview is paused instead). That preview
|
||||
is computed entirely in the browser (`src/lib/grading.ts` mirrors `trim.py`'s
|
||||
`_apply_grade` pixel math) as the sliders move, so it never waits on the
|
||||
server -- only **Apply** does that, to actually re-render the file. **Previous
|
||||
frame**/**Next frame** step the preview by exactly one frame (at the clip's
|
||||
own fps), for lining up a cut precisely -- the native scrub bar alone isn't
|
||||
frame-accurate. **Merge with next** undoes a cut, keeping the earlier of the
|
||||
two sections' grade.
|
||||
Sections are always contiguous and cover the whole kept range -- there's no
|
||||
reordering or gaps, only where to cut and what to grade each side; a plain
|
||||
range trim with no cuts is just the one-section case. Audio is one continuous
|
||||
track regardless of how many sections there are.
|
||||
|
||||
The untouched original is always kept alongside it the first time a clip is
|
||||
edited, and **every edit always re-renders from that original** with the full
|
||||
set of values shown in the dialog -- range and sections together, in one pass.
|
||||
That's what lets adjusting either one leave the other in place instead of each
|
||||
one silently discarding the other. The job itself records the range/sections
|
||||
of its last edit (`edit_start_seconds`/`edit_end_seconds`/`edit_segments`,
|
||||
alongside `output_path`), so reopening the dialog -- even after closing it or
|
||||
reloading the page -- shows exactly what produced the current video, not a
|
||||
guess: a never-edited clip shows one neutral section covering its full length;
|
||||
an edited one shows every section that made it, cuts and start offset
|
||||
included. **Restore original** undoes every edit in one step and resets those
|
||||
recorded values back to one neutral section too.
|
||||
|
||||
### API Endpoints
|
||||
|
||||
| Method | Path | Description |
|
||||
@@ -98,6 +188,15 @@ Refer to `.env.example` for reference.
|
||||
| `POST` | `/api/jobs` | Create a new job |
|
||||
| `POST` | `/api/jobs/{id}/start` | Start a pending/stopped/failed job |
|
||||
| `POST` | `/api/jobs/{id}/stop` | Request a running job to stop |
|
||||
| `POST` | `/api/jobs/queue` | Queue jobs (`{"job_ids": [...]}`), in order |
|
||||
| `POST` | `/api/jobs/{id}/queue` | Queue one job |
|
||||
| `POST` | `/api/jobs/{id}/dequeue` | Take a queued job back out of the queue |
|
||||
| `POST` | `/api/jobs/{id}/last-frame` | Save the job's last frame, for another job to start from |
|
||||
| `POST` | `/api/jobs/{id}/last-clip` | Save the job's trailing ~1s, for another job to continue from |
|
||||
| `POST` | `/api/scenes/merge` | Join finished jobs' videos (`{"job_ids": [...], "name": ""}`) in order |
|
||||
| `POST` | `/api/jobs/{id}/trim` | Cut a job's video to a range and/or grade it section by section (`{"start_seconds": 0, "end_seconds": null, "segments": [{"end_seconds": null, "brightness": 0, "contrast": 1, "saturation": 1}]}`), in place |
|
||||
| `POST` | `/api/jobs/{id}/restore-video`| Undo every edit, back to the original video |
|
||||
| `GET` | `/api/merged/{filename}` | Stream a merged scene video |
|
||||
| `DELETE` | `/api/jobs/{id}` | Delete a job |
|
||||
| `GET` | `/api/jobs/{id}/video` | Stream the generated video/image |
|
||||
| `GET` | `/api/jobs/{id}/logs` | Get job logs (polling, supports `?after=`) |
|
||||
|
||||
@@ -110,6 +110,15 @@ def _migrate_db(conn: sqlite3.Connection) -> None:
|
||||
_add_column_if_missing(conn, "jobs", "generator_update_interval", "INTEGER", "5")
|
||||
_add_column_if_missing(conn, "jobs", "real_score_model_path", "TEXT", "''")
|
||||
_add_column_if_missing(conn, "jobs", "fake_score_model_path", "TEXT", "''")
|
||||
_add_column_if_missing(conn, "jobs", "queued_at", "REAL", "NULL")
|
||||
_add_column_if_missing(conn, "jobs", "edit_start_seconds", "REAL", "0.0")
|
||||
_add_column_if_missing(conn, "jobs", "edit_end_seconds", "REAL", "NULL")
|
||||
# Superseded by edit_segments_json (a list of razor-cut sections, each with
|
||||
# its own grade); left in place, unused, rather than risking a DROP COLUMN.
|
||||
_add_column_if_missing(conn, "jobs", "edit_brightness", "REAL", "0.0")
|
||||
_add_column_if_missing(conn, "jobs", "edit_contrast", "REAL", "1.0")
|
||||
_add_column_if_missing(conn, "jobs", "edit_saturation", "REAL", "1.0")
|
||||
_add_column_if_missing(conn, "jobs", "edit_segments_json", "TEXT", "'[]'")
|
||||
# Settings table
|
||||
_add_column_if_missing(conn, "settings", "vae_cpu_offload", "INTEGER", "0")
|
||||
_add_column_if_missing(conn, "settings", "image_encoder_cpu_offload", "INTEGER", "0")
|
||||
@@ -326,6 +335,7 @@ class Database:
|
||||
return
|
||||
allowed = {
|
||||
"status",
|
||||
"queued_at",
|
||||
"started_at",
|
||||
"finished_at",
|
||||
"error",
|
||||
@@ -345,6 +355,43 @@ class Database:
|
||||
self._execute(sql, tuple(vals))
|
||||
self._commit()
|
||||
|
||||
#: Job fields stored under a different column name, and how they are encoded.
|
||||
_CONFIG_COLUMNS = {"references": "references_json", "edit_segments": "edit_segments_json"}
|
||||
_JSON_LIST_CONFIG_FIELDS = {"references", "edit_segments"}
|
||||
_BOOL_CONFIG_FIELDS = {
|
||||
"dit_cpu_offload",
|
||||
"text_encoder_cpu_offload",
|
||||
"vae_cpu_offload",
|
||||
"image_encoder_cpu_offload",
|
||||
"use_fsdp_inference",
|
||||
"enable_torch_compile",
|
||||
"dmd_use_vsa",
|
||||
}
|
||||
|
||||
def update_job_config(self, job_id: str, updates: dict[str, Any]) -> None:
|
||||
"""Persist edits to a job's configuration (prompt, references, sizes...).
|
||||
|
||||
Fields the table has no column for are skipped, as they are on insert.
|
||||
"""
|
||||
columns = _get_table_columns(self._conn(), "jobs")
|
||||
cols = []
|
||||
vals: list[Any] = []
|
||||
for key, value in updates.items():
|
||||
column = self._CONFIG_COLUMNS.get(key, key)
|
||||
if column not in columns or column == "id":
|
||||
continue
|
||||
if key in self._JSON_LIST_CONFIG_FIELDS:
|
||||
value = json.dumps(value or [])
|
||||
elif key in self._BOOL_CONFIG_FIELDS:
|
||||
value = 1 if value else 0
|
||||
cols.append(f"{column} = ?")
|
||||
vals.append(value)
|
||||
if not cols:
|
||||
return
|
||||
vals.append(job_id)
|
||||
self._execute(f"UPDATE jobs SET {', '.join(cols)} WHERE id = ?", tuple(vals))
|
||||
self._commit()
|
||||
|
||||
def delete_job(self, job_id: str) -> bool:
|
||||
"""Delete a job. Returns True if a row was deleted."""
|
||||
cur = self._execute("DELETE FROM jobs WHERE id = ?", (job_id, ))
|
||||
@@ -537,8 +584,31 @@ def _row_to_dataset(row: sqlite3.Row) -> dict[str, Any]:
|
||||
}
|
||||
|
||||
|
||||
def _row_to_job(row: sqlite3.Row) -> dict[str, Any]:
|
||||
class _ColumnRow:
|
||||
"""A sqlite3.Row whose ``in`` tests column names.
|
||||
|
||||
sqlite3.Row's own ``in`` tests *values*, so ``"fps" in row`` is always False
|
||||
and a job restored from the database silently lost fps, negative_prompt and
|
||||
the other optional columns.
|
||||
"""
|
||||
|
||||
def __init__(self, row: sqlite3.Row):
|
||||
self._row = row
|
||||
self._columns = set(row.keys())
|
||||
|
||||
def __getitem__(self, key: str) -> Any:
|
||||
return self._row[key]
|
||||
|
||||
def __contains__(self, key: object) -> bool:
|
||||
return key in self._columns
|
||||
|
||||
def keys(self) -> list[str]:
|
||||
return self._row.keys()
|
||||
|
||||
|
||||
def _row_to_job(sqlite_row: sqlite3.Row) -> dict[str, Any]:
|
||||
"""Convert a DB row to job dict (snake_case, matching Job.to_dict)."""
|
||||
row = _ColumnRow(sqlite_row)
|
||||
int_defaults = {
|
||||
"max_train_steps": 1000,
|
||||
"train_batch_size": 1,
|
||||
@@ -562,6 +632,7 @@ def _row_to_job(row: sqlite3.Row) -> dict[str, Any]:
|
||||
"job_type": _sqlite_row_get(row, "job_type", "inference"),
|
||||
"status": row["status"],
|
||||
"created_at": row["created_at"],
|
||||
"queued_at": row["queued_at"] if "queued_at" in row else None,
|
||||
"started_at": row["started_at"],
|
||||
"finished_at": row["finished_at"],
|
||||
"error": row["error"],
|
||||
@@ -614,6 +685,13 @@ def _row_to_job(row: sqlite3.Row) -> dict[str, Any]:
|
||||
for col in ("dmd_denoising_steps", "real_score_model_path", "fake_score_model_path"):
|
||||
if col in row:
|
||||
result[col] = (row[col] or "") or ""
|
||||
if "edit_start_seconds" in row:
|
||||
start_val: Any = row["edit_start_seconds"]
|
||||
result["edit_start_seconds"] = float(start_val) if start_val is not None else 0.0
|
||||
if "edit_end_seconds" in row:
|
||||
end_val: Any = row["edit_end_seconds"]
|
||||
result["edit_end_seconds"] = float(end_val) if end_val is not None else None
|
||||
result["edit_segments"] = _sqlite_row_get(row, "edit_segments_json", "") or ""
|
||||
return result
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,146 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Save the end of a finished clip, so the next clip can continue from it.
|
||||
|
||||
Two forms: the final frame alone (a still, for a hard "opens on this exact
|
||||
picture" continuation), or a short trailing clip (a few hundred milliseconds of
|
||||
video, attached as an H3 video reference so the model has actual motion to
|
||||
continue rather than guessing it from one still). Kept apart from ``server.py``
|
||||
because it needs nothing from FastVideo -- only PyAV -- and so can be exercised
|
||||
without a GPU.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from fractions import Fraction
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
VIDEO_EXTENSIONS = {".mp4", ".webm", ".avi", ".mov", ".mkv"}
|
||||
|
||||
|
||||
class FrameError(Exception):
|
||||
"""A request that can't be satisfied, carrying the HTTP status to report."""
|
||||
|
||||
def __init__(self, status_code: int, detail: str):
|
||||
super().__init__(detail)
|
||||
self.status_code = status_code
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def save_last_frame(video_path: str, out_path: str) -> None:
|
||||
"""Decode ``video_path`` and write its final frame to ``out_path`` as a PNG.
|
||||
|
||||
Clips are a few hundred frames, so decoding through to the end is simpler
|
||||
and more reliable than seeking (which lands on keyframes, not the last frame).
|
||||
"""
|
||||
import av
|
||||
|
||||
tmp_path = f"{out_path}.tmp.png"
|
||||
with av.open(video_path) as container:
|
||||
last = None
|
||||
for frame in container.decode(container.streams.video[0]):
|
||||
last = frame
|
||||
if last is None:
|
||||
raise ValueError("The video has no frames.")
|
||||
last.to_image().save(tmp_path)
|
||||
os.replace(tmp_path, out_path) # atomic, so a reader never sees a half-written file
|
||||
|
||||
|
||||
def last_frame_for_job(job: Any, upload_dir: str) -> str:
|
||||
"""Path of a PNG of the last frame of ``job``'s video, extracting it if needed.
|
||||
|
||||
Raises :class:`FrameError` with the status the API should return.
|
||||
"""
|
||||
status = getattr(job.status, "value", job.status)
|
||||
if status != "completed" or not job.output_path:
|
||||
raise FrameError(404, "No output available for this job")
|
||||
if not os.path.isfile(job.output_path):
|
||||
raise FrameError(404, "Output file not found on disk")
|
||||
if Path(job.output_path).suffix.lower() not in VIDEO_EXTENSIONS:
|
||||
raise FrameError(400, "This job's output is not a video")
|
||||
if not upload_dir:
|
||||
raise FrameError(503, "Upload directory not configured")
|
||||
|
||||
out_dir = os.path.join(upload_dir, "last_frames")
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
out_path = os.path.abspath(os.path.join(out_dir, f"last_frame_{job.id}.png"))
|
||||
# Reuse an earlier extraction unless the video has been rewritten since.
|
||||
if not (os.path.isfile(out_path) and os.path.getmtime(out_path) >= os.path.getmtime(job.output_path)):
|
||||
try:
|
||||
save_last_frame(job.output_path, out_path)
|
||||
except Exception as e:
|
||||
raise FrameError(500, f"Could not read the last frame: {e}") from e
|
||||
return out_path
|
||||
|
||||
|
||||
DEFAULT_LAST_CLIP_SECONDS = 1.0
|
||||
|
||||
|
||||
def save_last_clip(video_path: str, out_path: str, seconds: float = DEFAULT_LAST_CLIP_SECONDS) -> None:
|
||||
"""Decode the final ``seconds`` of ``video_path`` and re-encode them to ``out_path``.
|
||||
|
||||
Re-encoded rather than stream-copied: the source's keyframes rarely align
|
||||
with an arbitrary cut ``seconds`` from the end, and a copy can only start on
|
||||
one. Silent -- the trailing video is for motion continuity; voice continuity
|
||||
already comes from each job's own audio reference.
|
||||
"""
|
||||
import av
|
||||
|
||||
if seconds <= 0:
|
||||
raise ValueError(f"seconds must be positive, got {seconds}.")
|
||||
|
||||
tmp_path = f"{out_path}.tmp.mp4"
|
||||
with av.open(video_path) as src:
|
||||
video_stream = src.streams.video[0]
|
||||
fps = float(video_stream.average_rate or video_stream.guessed_rate or 24)
|
||||
frames = list(src.decode(video_stream))
|
||||
if not frames:
|
||||
raise ValueError("The video has no frames.")
|
||||
keep = max(1, round(seconds * fps))
|
||||
tail = frames[-keep:]
|
||||
|
||||
with av.open(tmp_path, mode="w") as out:
|
||||
out_stream = out.add_stream("h264", rate=round(fps))
|
||||
out_stream.width, out_stream.height = tail[0].width, tail[0].height
|
||||
out_stream.pix_fmt = "yuv420p"
|
||||
# Each frame needs its own increasing pts at a known time base, or PyAV
|
||||
# stamps every frame identically (pts=0), which corrupts the muxed
|
||||
# container's average-frame-rate metadata -- H3's reference pipeline
|
||||
# trusts that metadata to resample the clip, so a wrong rate there
|
||||
# silently collapses a short clip down to as little as one frame.
|
||||
frame_time_base = Fraction(1, round(fps))
|
||||
for i, frame in enumerate(tail):
|
||||
frame.pts = i
|
||||
frame.time_base = frame_time_base
|
||||
for packet in out_stream.encode(frame):
|
||||
out.mux(packet)
|
||||
for packet in out_stream.encode():
|
||||
out.mux(packet)
|
||||
os.replace(tmp_path, out_path) # atomic, so a reader never sees a half-written file
|
||||
|
||||
|
||||
def last_clip_for_job(job: Any, upload_dir: str, seconds: float = DEFAULT_LAST_CLIP_SECONDS) -> str:
|
||||
"""Path of a short video of the end of ``job``'s video, extracting it if needed.
|
||||
|
||||
Raises :class:`FrameError` with the status the API should return.
|
||||
"""
|
||||
status = getattr(job.status, "value", job.status)
|
||||
if status != "completed" or not job.output_path:
|
||||
raise FrameError(404, "No output available for this job")
|
||||
if not os.path.isfile(job.output_path):
|
||||
raise FrameError(404, "Output file not found on disk")
|
||||
if Path(job.output_path).suffix.lower() not in VIDEO_EXTENSIONS:
|
||||
raise FrameError(400, "This job's output is not a video")
|
||||
if not upload_dir:
|
||||
raise FrameError(503, "Upload directory not configured")
|
||||
|
||||
out_dir = os.path.join(upload_dir, "last_clips")
|
||||
os.makedirs(out_dir, exist_ok=True)
|
||||
out_path = os.path.abspath(os.path.join(out_dir, f"last_clip_{job.id}.mp4"))
|
||||
if not (os.path.isfile(out_path) and os.path.getmtime(out_path) >= os.path.getmtime(job.output_path)):
|
||||
try:
|
||||
save_last_clip(job.output_path, out_path, seconds)
|
||||
except Exception as e:
|
||||
raise FrameError(500, f"Could not read the end of the clip: {e}") from e
|
||||
return out_path
|
||||
@@ -0,0 +1,208 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Scheduling rules for queued jobs, and references to another job's ending.
|
||||
|
||||
Kept apart from ``job_runner.py`` because it is pure -- it works on plain
|
||||
snapshots and needs neither FastVideo nor a GPU -- so it can be tested anywhere.
|
||||
|
||||
A job can start from how another job's video ended -- either its last frame (a
|
||||
still, for a hard "opens on this exact picture" continuation) or its last
|
||||
second or so (a short video, so the model has actual motion to continue, not
|
||||
just a pose to guess motion from). Neither exists until the other job
|
||||
finishes, so the reference is *deferred*: its source is ``job-last-frame:<job
|
||||
id>`` or ``job-last-clip:<job id>``, resolved to a real file only when the job
|
||||
runs. That link is also the queue's dependency graph: a queued job waits until
|
||||
every job it takes an ending from has completed.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from collections.abc import Callable, Iterable, Mapping
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any, Protocol
|
||||
|
||||
DEFERRED_LAST_FRAME_PREFIX = "job-last-frame:"
|
||||
DEFERRED_LAST_CLIP_PREFIX = "job-last-clip:"
|
||||
_DEFERRED_PREFIXES = (DEFERRED_LAST_FRAME_PREFIX, DEFERRED_LAST_CLIP_PREFIX)
|
||||
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
PENDING = "pending"
|
||||
FAILED = "failed"
|
||||
STOPPED = "stopped"
|
||||
|
||||
|
||||
def deferred_last_frame_source(job_id: str) -> str:
|
||||
return f"{DEFERRED_LAST_FRAME_PREFIX}{job_id}"
|
||||
|
||||
|
||||
def deferred_last_clip_source(job_id: str) -> str:
|
||||
return f"{DEFERRED_LAST_CLIP_PREFIX}{job_id}"
|
||||
|
||||
|
||||
def dependency_of(reference: Mapping[str, Any] | None) -> str | None:
|
||||
"""The job a deferred reference takes its ending from, or None for any other."""
|
||||
source = (reference or {}).get("source")
|
||||
if not isinstance(source, str):
|
||||
return None
|
||||
for prefix in _DEFERRED_PREFIXES:
|
||||
if source.startswith(prefix):
|
||||
return source[len(prefix):] or None
|
||||
return None
|
||||
|
||||
|
||||
def dependencies(references: Iterable[Mapping[str, Any]] | None) -> list[str]:
|
||||
"""Job ids a job needs finished first, in reference order, without repeats."""
|
||||
found: list[str] = []
|
||||
for ref in references or []:
|
||||
dep = dependency_of(ref)
|
||||
if dep and dep not in found:
|
||||
found.append(dep)
|
||||
return found
|
||||
|
||||
|
||||
class JobView(Protocol):
|
||||
"""The parts of a job the scheduler looks at."""
|
||||
|
||||
id: str
|
||||
references: Any
|
||||
queued_at: float | None
|
||||
|
||||
@property
|
||||
def status(self) -> Any: # a JobStatus (str enum) or plain str
|
||||
...
|
||||
|
||||
|
||||
def _status(job: Any) -> str:
|
||||
return str(getattr(job.status, "value", job.status))
|
||||
|
||||
|
||||
def circular_dependency(jobs: Mapping[str, JobView], job_id: str) -> list[str] | None:
|
||||
"""The chain of job ids if ``job_id`` (transitively) waits on itself, else None."""
|
||||
|
||||
def walk(current: str, path: list[str]) -> list[str] | None:
|
||||
job = jobs.get(current)
|
||||
if job is None:
|
||||
return None
|
||||
for dep in dependencies(job.references):
|
||||
if dep == job_id:
|
||||
return [*path, current, dep]
|
||||
if dep in path or dep == current:
|
||||
continue
|
||||
found = walk(dep, [*path, current])
|
||||
if found:
|
||||
return found
|
||||
return None
|
||||
|
||||
return walk(job_id, [])
|
||||
|
||||
|
||||
def _label(job: Any) -> str:
|
||||
name = getattr(job, "name", "") or ""
|
||||
return f"'{name}'" if name else job.id
|
||||
|
||||
|
||||
@dataclass
|
||||
class Plan:
|
||||
"""What to do now: jobs to launch, and jobs that can never run."""
|
||||
|
||||
start: list[str] = field(default_factory=list)
|
||||
fail: dict[str, str] = field(default_factory=dict) # job id -> reason
|
||||
|
||||
|
||||
def plan_dispatch(jobs: Mapping[str, JobView], max_concurrent: int) -> Plan:
|
||||
"""Decide which queued jobs launch now and which have become impossible.
|
||||
|
||||
* Queued jobs run oldest-queued first, at most ``max_concurrent`` at a time
|
||||
(counting jobs already running).
|
||||
* A job whose frame source hasn't completed *waits*, and doesn't hold up
|
||||
jobs behind it that don't need it.
|
||||
* A job whose frame source failed, was stopped, or was deleted can never
|
||||
run, so it fails -- and so does anything waiting on *it*.
|
||||
* A job whose frame source is idle (neither queued nor running) keeps
|
||||
waiting: the user may still queue it.
|
||||
"""
|
||||
plan = Plan()
|
||||
status = {jid: _status(job) for jid, job in jobs.items()}
|
||||
queued = sorted(
|
||||
(job for job in jobs.values() if status[job.id] == QUEUED),
|
||||
key=lambda job: (job.queued_at or 0.0, job.id),
|
||||
)
|
||||
|
||||
# Failures cascade down a chain, so repeat until nothing new fails.
|
||||
changed = True
|
||||
while changed:
|
||||
changed = False
|
||||
for job in queued:
|
||||
if job.id in plan.fail:
|
||||
continue
|
||||
for dep_id in dependencies(job.references):
|
||||
dep = jobs.get(dep_id)
|
||||
if dep is None:
|
||||
plan.fail[job.id] = "Waiting on the ending of a job that was deleted."
|
||||
elif dep_id in plan.fail or status[dep_id] in (FAILED, STOPPED):
|
||||
reason = "failed" if dep_id in plan.fail or status[dep_id] == FAILED else "was stopped"
|
||||
plan.fail[job.id] = f"Needs the ending of {_label(dep)}, which {reason}."
|
||||
else:
|
||||
continue
|
||||
status[job.id] = FAILED
|
||||
changed = True
|
||||
break
|
||||
|
||||
slots = max(max_concurrent, 1) - sum(1 for s in status.values() if s == RUNNING)
|
||||
for job in queued:
|
||||
if slots <= 0:
|
||||
break
|
||||
if job.id in plan.fail:
|
||||
continue
|
||||
if all(status.get(dep) == COMPLETED for dep in dependencies(job.references)):
|
||||
plan.start.append(job.id)
|
||||
slots -= 1
|
||||
return plan
|
||||
|
||||
|
||||
class UnresolvedReference(ValueError):
|
||||
"""A deferred reference can't be turned into a real file."""
|
||||
|
||||
|
||||
def resolve_references(
|
||||
references: list[dict[str, Any]] | None,
|
||||
get_job: Callable[[str], Any],
|
||||
last_frame: Callable[[Any], str],
|
||||
last_clip: Callable[[Any], str] | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Copy of ``references`` with each deferred reference made real.
|
||||
|
||||
``last_frame(job)`` and ``last_clip(job)`` each return the path of an
|
||||
extracted file and may raise (see ``frames.FrameError``); the message is
|
||||
passed on. ``last_clip`` is optional so callers that never attach a
|
||||
trailing-clip reference (e.g. the mock server) don't need to supply one --
|
||||
it is only required if a ``job-last-clip:`` reference is actually present.
|
||||
Other references are returned unchanged, and the stored list is never
|
||||
modified, so a re-run resolves again against the source job's current
|
||||
output.
|
||||
"""
|
||||
resolved: list[dict[str, Any]] = []
|
||||
for ref in references or []:
|
||||
source = (ref or {}).get("source")
|
||||
prefix = next((p for p in _DEFERRED_PREFIXES if isinstance(source, str) and source.startswith(p)), None)
|
||||
if prefix is None:
|
||||
resolved.append(ref)
|
||||
continue
|
||||
dep_id = source[len(prefix):] or None
|
||||
extract = last_frame if prefix == DEFERRED_LAST_FRAME_PREFIX else last_clip
|
||||
what = "the last frame of" if prefix == DEFERRED_LAST_FRAME_PREFIX else "the ending of"
|
||||
if extract is None:
|
||||
raise UnresolvedReference("This server can't resolve a trailing-clip reference yet.")
|
||||
dep = get_job(dep_id) if dep_id else None
|
||||
if dep is None:
|
||||
raise UnresolvedReference(f"A reference needs {what} a job that no longer exists.")
|
||||
if _status(dep) != COMPLETED:
|
||||
raise UnresolvedReference(f"A reference needs {what} {_label(dep)}, which hasn't finished "
|
||||
f"(it is {_status(dep)}).")
|
||||
try:
|
||||
path = extract(dep)
|
||||
except Exception as exc: # FrameError carries a readable .detail
|
||||
raise UnresolvedReference(str(getattr(exc, "detail", None) or exc)) from exc
|
||||
resolved.append({**ref, "source": path})
|
||||
return resolved
|
||||
@@ -17,6 +17,7 @@ import logging
|
||||
import logging.handlers
|
||||
import multiprocessing as mp
|
||||
import os
|
||||
import queue
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
@@ -29,6 +30,15 @@ import yaml
|
||||
|
||||
from fastvideo.utils import get_mp_context
|
||||
from fastvideo_studio.database import Database
|
||||
from fastvideo_studio.frames import last_clip_for_job, last_frame_for_job
|
||||
from fastvideo_studio.trim import restore_original, trim_video
|
||||
from fastvideo_studio.job_queue import (
|
||||
UnresolvedReference,
|
||||
circular_dependency,
|
||||
dependencies,
|
||||
plan_dispatch,
|
||||
resolve_references,
|
||||
)
|
||||
from fastvideo_studio.training_config import (
|
||||
build_training_config,
|
||||
get_training_env,
|
||||
@@ -46,6 +56,7 @@ _MAX_LOG_LINES = 2000 # ring-buffer cap per job
|
||||
|
||||
class JobStatus(str, enum.Enum):
|
||||
PENDING = "pending"
|
||||
QUEUED = "queued"
|
||||
RUNNING = "running"
|
||||
COMPLETED = "completed"
|
||||
FAILED = "failed"
|
||||
@@ -138,6 +149,16 @@ class Job:
|
||||
finished_at: float | None = None
|
||||
error: str | None = None
|
||||
output_path: str | None = None
|
||||
queued_at: float | None = None
|
||||
# The range/sections last applied via trim_job(), so the Edit video dialog
|
||||
# can reopen showing what actually produced the current output_path instead
|
||||
# of resetting to neutral. Reset to these defaults by restore_job_video().
|
||||
edit_start_seconds: float = 0.0
|
||||
edit_end_seconds: float | None = None
|
||||
#: Ordered, contiguous razor-cut sections, each a {end_seconds, brightness,
|
||||
#: contrast, saturation} dict; see trim.py's DEFAULT_SEGMENTS. Empty means
|
||||
#: never edited (or restored) -- one neutral section covering everything.
|
||||
edit_segments: list[dict[str, Any]] = field(default_factory=list)
|
||||
log_file_path: str | None = None # Path to the job's log file
|
||||
num_inference_steps: int = 50
|
||||
num_frames: int = 81
|
||||
@@ -195,11 +216,15 @@ class Job:
|
||||
"references": self.references,
|
||||
"status": self.status.value,
|
||||
"created_at": self.created_at,
|
||||
"queued_at": self.queued_at,
|
||||
"started_at": self.started_at,
|
||||
"finished_at": self.finished_at,
|
||||
"error": self.error,
|
||||
"output_path": self.output_path,
|
||||
"log_file_path": self.log_file_path,
|
||||
"edit_start_seconds": self.edit_start_seconds,
|
||||
"edit_end_seconds": self.edit_end_seconds,
|
||||
"edit_segments": self.edit_segments,
|
||||
"num_inference_steps": self.num_inference_steps,
|
||||
"num_frames": self.num_frames,
|
||||
"height": self.height,
|
||||
@@ -278,8 +303,8 @@ def _job_log_path(output_dir: str, job_id: str) -> str:
|
||||
return os.path.join(output_dir, job_id, JOB_LOG_FILENAME)
|
||||
|
||||
|
||||
def _decode_references(value: Any) -> list[dict[str, Any]]:
|
||||
"""Reference lists round-trip through the DB as JSON text."""
|
||||
def _decode_json_list(value: Any) -> list[dict[str, Any]]:
|
||||
"""References and edit_segments both round-trip through the DB as JSON text."""
|
||||
if not value:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
@@ -287,7 +312,7 @@ def _decode_references(value: Any) -> list[dict[str, Any]]:
|
||||
try:
|
||||
decoded = json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
logger.warning("Could not decode stored references: %r", value)
|
||||
logger.warning("Could not decode stored JSON list: %r", value)
|
||||
return []
|
||||
return list(decoded) if isinstance(decoded, list) else []
|
||||
|
||||
@@ -320,6 +345,8 @@ class JobRunner:
|
||||
log_dir: str,
|
||||
database: Database,
|
||||
verbose: bool = False,
|
||||
max_concurrent_jobs: int = 1,
|
||||
upload_dir: str = "",
|
||||
):
|
||||
"""Initialize the job runner.
|
||||
|
||||
@@ -328,19 +355,38 @@ class JobRunner:
|
||||
log_dir: Directory where job log files are saved
|
||||
verbose: Whether to print full tracebacks in error messages
|
||||
database: Optional SQLite database for job persistence
|
||||
max_concurrent_jobs: How many *queued* jobs may run at once. Jobs
|
||||
started directly with start_job() run immediately regardless
|
||||
(but count toward the limit, so the queue waits for them).
|
||||
upload_dir: Where extracted last frames are saved.
|
||||
"""
|
||||
self.output_dir = output_dir
|
||||
self.log_dir = log_dir
|
||||
self.verbose = verbose
|
||||
self._db = database
|
||||
self.max_concurrent_jobs = max(max_concurrent_jobs, 1)
|
||||
self.upload_dir = upload_dir or os.path.join(output_dir, "uploads")
|
||||
|
||||
self._jobs: dict[str, Job] = {}
|
||||
self._jobs_lock = threading.Lock()
|
||||
# Serializes queue decisions (enqueue / dequeue / dispatch) so two
|
||||
# finishing jobs can't both take the same free slot.
|
||||
self._queue_lock = threading.RLock()
|
||||
self._last_queued_at = 0.0
|
||||
self._load_jobs()
|
||||
|
||||
# Cache of loaded generators keyed by model config so that we only pay
|
||||
# the model-loading cost once per model configuration.
|
||||
self._generators: dict[tuple, Any] = {}
|
||||
# Cached generators are keyed by (owning thread, config). The model's GPU
|
||||
# worker processes are killed (PR_SET_PDEATHSIG) when the *thread* that
|
||||
# started them exits, so a generator is only safe for that thread to reuse
|
||||
# -- which is why inference jobs run on long-lived worker threads (below)
|
||||
# rather than a new thread each.
|
||||
self._generator_threads: dict[tuple, threading.Thread] = {}
|
||||
# Inference jobs are handed to these threads through one shared queue.
|
||||
self._work: queue.Queue[Job | None] = queue.Queue()
|
||||
self._workers: list[threading.Thread] = []
|
||||
self._generators_lock = threading.Lock()
|
||||
|
||||
# Shared Manager for log queues (avoids spawning a new process per job)
|
||||
@@ -351,6 +397,14 @@ class JobRunner:
|
||||
os.makedirs(self.output_dir, exist_ok=True)
|
||||
os.makedirs(self.log_dir, exist_ok=True)
|
||||
|
||||
for i in range(self.max_concurrent_jobs):
|
||||
worker = threading.Thread(target=self._worker_loop, name=f"inference-worker-{i}", daemon=True)
|
||||
worker.start()
|
||||
self._workers.append(worker)
|
||||
|
||||
# Jobs that were queued when the server last stopped carry on.
|
||||
self._dispatch()
|
||||
|
||||
def _load_logs(self, job: Job) -> None:
|
||||
"""Populate job's log buffer from its log file if it exists."""
|
||||
path = job.log_file_path
|
||||
@@ -399,7 +453,7 @@ class JobRunner:
|
||||
job_type=row.get("job_type", "inference"),
|
||||
image_path=row.get("image_path", "") or "",
|
||||
last_image_path=row.get("last_image_path", "") or "",
|
||||
references=_decode_references(row.get("references")),
|
||||
references=_decode_json_list(row.get("references")),
|
||||
data_path=row.get("data_path", "") or "",
|
||||
max_train_steps=row.get("max_train_steps", 1000),
|
||||
train_batch_size=row.get("train_batch_size", 1),
|
||||
@@ -416,11 +470,16 @@ class JobRunner:
|
||||
fake_score_model_path=row.get("fake_score_model_path", "") or "",
|
||||
status=JobStatus(status),
|
||||
created_at=row["created_at"],
|
||||
queued_at=row.get("queued_at"),
|
||||
started_at=row.get("started_at"),
|
||||
finished_at=row.get("finished_at"),
|
||||
error=row.get("error"),
|
||||
output_path=row.get("output_path"),
|
||||
log_file_path=row.get("log_file_path"),
|
||||
# `or <default>` would be wrong here: 0.0 is a real, intentional value, not just "unset".
|
||||
edit_start_seconds=(float(row["edit_start_seconds"]) if row.get("edit_start_seconds") is not None else 0.0),
|
||||
edit_end_seconds=(float(row["edit_end_seconds"]) if row.get("edit_end_seconds") is not None else None),
|
||||
edit_segments=_decode_json_list(row.get("edit_segments")),
|
||||
num_inference_steps=row.get("num_inference_steps", 50),
|
||||
num_frames=row.get("num_frames", 81),
|
||||
height=row.get("height", 480),
|
||||
@@ -456,6 +515,7 @@ class JobRunner:
|
||||
self._db.update_job(
|
||||
job.id, {
|
||||
"status": job.status.value,
|
||||
"queued_at": job.queued_at,
|
||||
"started_at": job.started_at,
|
||||
"finished_at": job.finished_at,
|
||||
"error": job.error,
|
||||
@@ -466,7 +526,9 @@ class JobRunner:
|
||||
logger.warning("Failed to persist job %s: %s", job.id, exc)
|
||||
|
||||
def _shutdown(self) -> None:
|
||||
"""Shutdown the shared multiprocessing manager on exit."""
|
||||
"""Stop the inference workers and the shared multiprocessing manager on exit."""
|
||||
for _ in self._workers:
|
||||
self._work.put(None)
|
||||
try:
|
||||
self._mp_manager.shutdown()
|
||||
except Exception as exc:
|
||||
@@ -610,6 +672,9 @@ class JobRunner:
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to delete job %s from database: %s", job_id, exc)
|
||||
logger.info("Deleted job %s", job.id)
|
||||
# Anything queued behind this job's last frame can no longer run, and a
|
||||
# freed queue slot may let the next job start.
|
||||
self._dispatch()
|
||||
return True
|
||||
|
||||
CONFIG_FIELDS: tuple[str, ...] = (
|
||||
@@ -688,24 +753,30 @@ class JobRunner:
|
||||
for field_name, value in updates.items():
|
||||
setattr(job, field_name, value)
|
||||
self._save_job(job)
|
||||
try:
|
||||
self._db.update_job_config(job.id, updates)
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to persist edits to job %s: %s", job.id, exc)
|
||||
return job
|
||||
|
||||
def start_job(self, job_id: str) -> Job:
|
||||
"""Start (or restart) a pending / stopped / failed job.
|
||||
|
||||
Raises:
|
||||
ValueError: If job not found or cannot be started
|
||||
"""
|
||||
#: Statuses a job can be started or queued from.
|
||||
STARTABLE_STATUSES = (JobStatus.PENDING, JobStatus.FAILED, JobStatus.STOPPED)
|
||||
|
||||
def _startable_job(self, job_id: str) -> Job:
|
||||
with self._jobs_lock:
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
if job.status == JobStatus.RUNNING:
|
||||
raise ValueError("Job is already running")
|
||||
if job.status == JobStatus.QUEUED:
|
||||
raise ValueError("Job is already queued")
|
||||
if job.status == JobStatus.COMPLETED:
|
||||
raise ValueError("Job already completed. Delete and re-create to run again.")
|
||||
# Reset state for re-run
|
||||
job.status = JobStatus.PENDING
|
||||
return job
|
||||
|
||||
def _reset_for_run(self, job: Job) -> None:
|
||||
"""Clear the previous attempt's state so the job starts clean."""
|
||||
job.error = None
|
||||
job.output_path = None
|
||||
job.log_file_path = None # Will be set when job starts
|
||||
@@ -716,26 +787,189 @@ class JobRunner:
|
||||
job.log_file_handler = None # Reset file handler
|
||||
job._process = None # Reset subprocess handle
|
||||
|
||||
# Wrap _run_job in an additional safety layer to catch any exceptions
|
||||
# that might escape (though they shouldn't with our comprehensive handling)
|
||||
def safe_run_job(job: Job):
|
||||
"""Wrapper to ensure _run_job never raises an unhandled exception."""
|
||||
try:
|
||||
self._run_job(job)
|
||||
except BaseException as exc:
|
||||
# This should never happen, but if it does, we catch it here
|
||||
logger.critical("Unhandled exception escaped from _run_job for job %s: %s", job.id, exc, exc_info=True)
|
||||
with contextlib.suppress(Exception):
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = (f"Unhandled exception: {type(exc).__name__}: {str(exc)}")
|
||||
job.finished_at = time.time()
|
||||
def _launch(self, job: Job) -> None:
|
||||
"""Start ``job``; when it ends, look for more queued work.
|
||||
|
||||
thread = threading.Thread(target=safe_run_job, args=(job, ), daemon=True)
|
||||
Inference jobs go to a long-lived worker thread so the loaded model
|
||||
survives from one job to the next. Training jobs only babysit a
|
||||
subprocess, so they get a thread of their own.
|
||||
"""
|
||||
# Marked running now, not when the job gets going, so the queue counts it
|
||||
# immediately and can't start one job too many.
|
||||
job.status = JobStatus.RUNNING
|
||||
job.started_at = time.time()
|
||||
self._save_job(job)
|
||||
|
||||
if job.job_type == "inference":
|
||||
self._work.put(job)
|
||||
logger.info("Sent job %s to an inference worker", job.id)
|
||||
return
|
||||
|
||||
def run_training_job(job: Job):
|
||||
self._run_job_safely(job)
|
||||
self._dispatch_after_exit(threading.current_thread())
|
||||
|
||||
thread = threading.Thread(target=run_training_job, args=(job, ), daemon=True)
|
||||
job._thread = thread
|
||||
thread.start()
|
||||
logger.info("Started job %s", job.id)
|
||||
|
||||
def _run_job_safely(self, job: Job) -> None:
|
||||
"""Run a job, turning anything that escapes into a failed job."""
|
||||
try:
|
||||
self._run_job(job)
|
||||
except BaseException as exc:
|
||||
# This should never happen, but if it does, we catch it here
|
||||
logger.critical("Unhandled exception escaped from _run_job for job %s: %s", job.id, exc, exc_info=True)
|
||||
with contextlib.suppress(Exception):
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = (f"Unhandled exception: {type(exc).__name__}: {str(exc)}")
|
||||
job.finished_at = time.time()
|
||||
self._save_job(job)
|
||||
|
||||
def _worker_loop(self) -> None:
|
||||
"""Run inference jobs one after another on this thread, for the life of the server.
|
||||
|
||||
The thread never exits between jobs. That matters: it started the model's
|
||||
GPU processes, which are killed when it exits, so staying alive is what
|
||||
lets the next job find the model already loaded.
|
||||
"""
|
||||
while True:
|
||||
job = self._work.get()
|
||||
if job is None:
|
||||
return
|
||||
self._run_job_safely(job)
|
||||
# The slot is free; dependents may now be runnable (or impossible).
|
||||
with contextlib.suppress(Exception):
|
||||
self._dispatch()
|
||||
|
||||
def start_job(self, job_id: str) -> Job:
|
||||
"""Start (or restart) a pending / stopped / failed job right now.
|
||||
|
||||
Refused if every inference worker is already busy; use enqueue_job() to
|
||||
wait for a free one. A job that takes another job's last frame can only
|
||||
start once that job has completed.
|
||||
|
||||
Raises:
|
||||
ValueError: If job not found or cannot be started
|
||||
"""
|
||||
with self._queue_lock:
|
||||
job = self._startable_job(job_id)
|
||||
# Fail here, before touching the job, rather than as a failed run.
|
||||
self._check_dependencies_finished(job)
|
||||
if job.job_type == "inference":
|
||||
busy = self._running_inference_jobs()
|
||||
if busy >= self.max_concurrent_jobs:
|
||||
raise ValueError(f"{busy} inference job(s) already running and the model workers are busy. "
|
||||
"Queue this job to run next.")
|
||||
self._reset_for_run(job)
|
||||
job.queued_at = None
|
||||
self._launch(job)
|
||||
return job
|
||||
|
||||
def _running_inference_jobs(self) -> int:
|
||||
with self._jobs_lock:
|
||||
return sum(1 for j in self._jobs.values() if j.job_type == "inference" and j.status == JobStatus.RUNNING)
|
||||
|
||||
def _check_dependencies_finished(self, job: Job) -> None:
|
||||
for dep_id in dependencies(job.references):
|
||||
dep = self.get_job(dep_id)
|
||||
if dep is None:
|
||||
raise ValueError("This job starts from the ending of a job that no longer exists.")
|
||||
if dep.status != JobStatus.COMPLETED:
|
||||
label = f"'{dep.name}'" if dep.name else dep.id
|
||||
raise ValueError(f"This job starts from the ending of {label}, which hasn't finished "
|
||||
f"(it is {dep.status.value}). Queue it to run after that job.")
|
||||
|
||||
# -- queue ---------------------------------------------------------------
|
||||
|
||||
def _next_queued_at(self) -> float:
|
||||
"""A strictly increasing timestamp, so jobs queued together keep their order."""
|
||||
self._last_queued_at = max(time.time(), self._last_queued_at + 1e-6)
|
||||
return self._last_queued_at
|
||||
|
||||
def enqueue_jobs(self, job_ids: list[str]) -> list[Job]:
|
||||
"""Queue jobs, in the given order, to run as slots free up.
|
||||
|
||||
All-or-nothing: if any job can't be queued, none are.
|
||||
|
||||
Raises:
|
||||
ValueError: If a job is missing, not startable, or waits on itself.
|
||||
"""
|
||||
if len(set(job_ids)) != len(job_ids):
|
||||
raise ValueError("A job was listed more than once.")
|
||||
with self._queue_lock:
|
||||
jobs = [self._startable_job(job_id) for job_id in job_ids]
|
||||
with self._jobs_lock:
|
||||
snapshot = dict(self._jobs)
|
||||
for job in jobs:
|
||||
loop = circular_dependency(snapshot, job.id)
|
||||
if loop:
|
||||
raise ValueError("Jobs would wait on each other's last frames in a loop: " + " -> ".join(loop))
|
||||
for job in jobs:
|
||||
self._reset_for_run(job)
|
||||
job.status = JobStatus.QUEUED
|
||||
job.queued_at = self._next_queued_at()
|
||||
self._save_job(job)
|
||||
logger.info("Queued job %s", job.id)
|
||||
self._dispatch()
|
||||
return jobs
|
||||
|
||||
def enqueue_job(self, job_id: str) -> Job:
|
||||
return self.enqueue_jobs([job_id])[0]
|
||||
|
||||
def dequeue_job(self, job_id: str) -> Job:
|
||||
"""Take a queued job out of the queue, back to pending.
|
||||
|
||||
Raises:
|
||||
ValueError: If the job is missing or isn't queued.
|
||||
"""
|
||||
with self._queue_lock:
|
||||
job = self.get_job(job_id)
|
||||
if job is None:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
if job.status != JobStatus.QUEUED:
|
||||
raise ValueError(f"Job is not queued (status={job.status.value})")
|
||||
job.status = JobStatus.PENDING
|
||||
job.queued_at = None
|
||||
self._save_job(job)
|
||||
logger.info("Dequeued job %s", job.id)
|
||||
# Anything queued behind this one's frame keeps waiting; a slot it held
|
||||
# in line may now go to another job.
|
||||
self._dispatch()
|
||||
return job
|
||||
|
||||
def _dispatch_after_exit(self, finished: threading.Thread) -> None:
|
||||
"""Wait for a job's thread to end, then start whatever is next in the queue."""
|
||||
finished.join()
|
||||
with contextlib.suppress(Exception):
|
||||
self._dispatch()
|
||||
|
||||
def _dispatch(self) -> None:
|
||||
"""Launch queued jobs that are ready, and fail those that can never be."""
|
||||
with self._queue_lock:
|
||||
with self._jobs_lock:
|
||||
snapshot = dict(self._jobs)
|
||||
plan = plan_dispatch(snapshot, self.max_concurrent_jobs)
|
||||
for job_id, reason in plan.fail.items():
|
||||
job = snapshot[job_id]
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = reason
|
||||
job.finished_at = time.time()
|
||||
self._save_job(job)
|
||||
logger.warning("Queued job %s can't run: %s", job_id, reason)
|
||||
for job_id in plan.start:
|
||||
job = snapshot[job_id]
|
||||
job.queued_at = None
|
||||
try:
|
||||
self._launch(job)
|
||||
except Exception as exc: # e.g. the OS refused a thread
|
||||
logger.exception("Could not launch queued job %s", job_id)
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = f"Could not start: {exc}"
|
||||
job.finished_at = time.time()
|
||||
self._save_job(job)
|
||||
|
||||
def stop_job(self, job_id: str) -> Job:
|
||||
"""Request a running job to stop.
|
||||
|
||||
@@ -749,6 +983,8 @@ class JobRunner:
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
if job.status == JobStatus.QUEUED:
|
||||
return self.dequeue_job(job_id)
|
||||
if job.status != JobStatus.RUNNING:
|
||||
raise ValueError(f"Job is not running (status={job.status.value})")
|
||||
|
||||
@@ -759,6 +995,91 @@ class JobRunner:
|
||||
logger.info("Stop requested for job %s", job.id)
|
||||
return job
|
||||
|
||||
# -- trimming --------------------------------------------------------------
|
||||
|
||||
def _set_output(
|
||||
self,
|
||||
job: Job,
|
||||
output_path: str,
|
||||
num_frames: int,
|
||||
*,
|
||||
edit_start_seconds: float = 0.0,
|
||||
edit_end_seconds: float | None = None,
|
||||
edit_segments: list[dict[str, Any]] | None = None,
|
||||
) -> None:
|
||||
"""Point a completed job at a different output file, e.g. after trimming.
|
||||
|
||||
The edit_* keyword args record what produced that file (left at their
|
||||
neutral/full-range defaults by restore_job_video()), so the Edit video
|
||||
dialog can reopen showing the edit actually in effect instead of always
|
||||
resetting to neutral -- nothing else needs these; other jobs' last-frame/
|
||||
last-clip references and a merge only ever read output_path.
|
||||
|
||||
Bypasses update_job_config()'s pending/failed/stopped gate on purpose:
|
||||
a completed job's *generation* config stays fixed, but which cut of its
|
||||
own video is showing is not a generation setting.
|
||||
"""
|
||||
edit_segments = edit_segments or []
|
||||
job.output_path = output_path
|
||||
job.num_frames = num_frames
|
||||
job.edit_start_seconds = edit_start_seconds
|
||||
job.edit_end_seconds = edit_end_seconds
|
||||
job.edit_segments = edit_segments
|
||||
self._save_job(job)
|
||||
try:
|
||||
self._db.update_job_config(
|
||||
job.id, {
|
||||
"num_frames": num_frames,
|
||||
"edit_start_seconds": edit_start_seconds,
|
||||
"edit_end_seconds": edit_end_seconds,
|
||||
"edit_segments": edit_segments,
|
||||
})
|
||||
except Exception as exc:
|
||||
logger.warning("Failed to persist edit state for job %s: %s", job.id, exc)
|
||||
|
||||
def trim_job(
|
||||
self,
|
||||
job_id: str,
|
||||
start_seconds: float = 0.0,
|
||||
end_seconds: float | None = None,
|
||||
segments: list[dict[str, Any]] | None = None,
|
||||
) -> Job:
|
||||
"""Cut a completed job's video to [start_seconds, end_seconds) and grade it section by section, in place.
|
||||
|
||||
The untouched original is kept, so this can be called again with a
|
||||
different range and/or sections -- it always renders from that original
|
||||
with the full set of parameters given, never compounding quality loss
|
||||
or discarding one edit by changing the other -- or undone with
|
||||
restore_job_video().
|
||||
"""
|
||||
with self._jobs_lock:
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
path, kept = trim_video(job, start_seconds, end_seconds, segments)
|
||||
self._set_output(
|
||||
job,
|
||||
path,
|
||||
kept,
|
||||
edit_start_seconds=start_seconds,
|
||||
edit_end_seconds=end_seconds,
|
||||
edit_segments=segments,
|
||||
)
|
||||
logger.info("Edited job %s: %.2f-%s s, %d section(s) (%d frames)", job.id, start_seconds,
|
||||
f"{end_seconds:.2f}" if end_seconds is not None else "end", len(segments or [1]), kept)
|
||||
return job
|
||||
|
||||
def restore_job_video(self, job_id: str) -> Job:
|
||||
"""Undo every trim on a job's video, back to what it originally generated."""
|
||||
with self._jobs_lock:
|
||||
job = self._jobs.get(job_id)
|
||||
if job is None:
|
||||
raise ValueError(f"Job {job_id} not found")
|
||||
path, kept = restore_original(job)
|
||||
self._set_output(job, path, kept)
|
||||
logger.info("Restored job %s to its original video", job.id)
|
||||
return job
|
||||
|
||||
def get_job_logs(self, job_id: str, after: int = 0) -> dict[str, Any]:
|
||||
"""Return log lines for a job.
|
||||
|
||||
@@ -786,6 +1107,22 @@ class JobRunner:
|
||||
"phase": job._log_buf.phase,
|
||||
}
|
||||
|
||||
def _discard_generators(self) -> None:
|
||||
"""Unload this thread's models so the next job on it starts from a clean load."""
|
||||
with self._generators_lock:
|
||||
for key in [k for k in self._generators if k[0] == threading.current_thread().ident]:
|
||||
self._drop_generator(key)
|
||||
|
||||
def _drop_generator(self, key: tuple) -> None:
|
||||
"""Forget a cached generator and shut it down. Call with ``_generators_lock`` held."""
|
||||
gen = self._generators.pop(key, None)
|
||||
self._generator_threads.pop(key, None)
|
||||
if gen is not None:
|
||||
try:
|
||||
gen.shutdown()
|
||||
except Exception:
|
||||
logger.debug("Shutdown of a cached generator failed", exc_info=True)
|
||||
|
||||
def _get_or_create_generator(
|
||||
self,
|
||||
model_id: str,
|
||||
@@ -823,23 +1160,27 @@ class JobRunner:
|
||||
sp_size,
|
||||
)
|
||||
|
||||
# Generators are cached by model_id and configuration parameters
|
||||
# Generators are cached per thread and configuration. Only the thread that
|
||||
# built one may reuse it: its GPU processes die with that thread.
|
||||
thread = threading.current_thread()
|
||||
entry_key = (thread.ident, cache_key)
|
||||
with self._generators_lock:
|
||||
cached = self._generators.get(cache_key)
|
||||
# Generators whose thread has exited have dead workers already.
|
||||
for key, owner in list(self._generator_threads.items()):
|
||||
if not owner.is_alive():
|
||||
self._drop_generator(key)
|
||||
cached = self._generators.get(entry_key)
|
||||
if cached is not None:
|
||||
if _generator_is_alive(cached):
|
||||
return cached
|
||||
# Workers can exit while a generator sits idle in the cache;
|
||||
# reusing it fails every later job with the same config.
|
||||
logger.warning(
|
||||
"Cached generator for %s has dead workers; reloading.",
|
||||
model_id,
|
||||
)
|
||||
self._generators.pop(cache_key, None)
|
||||
try:
|
||||
cached.shutdown()
|
||||
except Exception:
|
||||
logger.debug("Shutdown of the dead generator failed", exc_info=True)
|
||||
# Workers can exit while a generator sits idle in the cache.
|
||||
logger.warning("Cached generator for %s has dead workers; reloading.", model_id)
|
||||
self._drop_generator(entry_key)
|
||||
# A worker keeps one model resident. A different configuration replaces
|
||||
# it, freeing its GPU memory first, rather than loading beside it.
|
||||
for key in [k for k in self._generators if k[0] == thread.ident]:
|
||||
logger.info("Unloading the previous model to load %s.", model_id)
|
||||
self._drop_generator(key)
|
||||
|
||||
# Import lazily so starting the server is fast even without a GPU.
|
||||
from fastvideo import VideoGenerator
|
||||
@@ -883,11 +1224,8 @@ class JobRunner:
|
||||
)
|
||||
|
||||
with self._generators_lock:
|
||||
if cache_key not in self._generators:
|
||||
self._generators[cache_key] = gen
|
||||
else: # Another thread may have created it while we were loading.
|
||||
gen.shutdown()
|
||||
gen = self._generators[cache_key]
|
||||
self._generators[entry_key] = gen
|
||||
self._generator_threads[entry_key] = thread
|
||||
return gen
|
||||
|
||||
def _run_job(self, job: Job):
|
||||
@@ -1064,6 +1402,15 @@ class JobRunner:
|
||||
logger.warning("Job stopped before execution started")
|
||||
return
|
||||
|
||||
# Frames and trailing clips borrowed from other jobs are extracted now,
|
||||
# when they exist. Done before loading the model so a missing one fails fast.
|
||||
resolved_references = resolve_references(
|
||||
job.references,
|
||||
self.get_job,
|
||||
last_frame=lambda source: last_frame_for_job(source, self.upload_dir),
|
||||
last_clip=lambda source: last_clip_for_job(source, self.upload_dir),
|
||||
)
|
||||
|
||||
buf.phase = "loading model"
|
||||
logger.info("Loading model...")
|
||||
|
||||
@@ -1121,7 +1468,7 @@ class JobRunner:
|
||||
if job.image_path:
|
||||
gen_kwargs["image_path"] = job.image_path
|
||||
if job.references:
|
||||
gen_kwargs["references"] = _build_h3_references(job.references)
|
||||
gen_kwargs["references"] = _build_h3_references(resolved_references)
|
||||
if job.last_image_path:
|
||||
# _prepare_fl2va requires a PIL image, not a path.
|
||||
from PIL import Image as _PILImage
|
||||
@@ -1148,6 +1495,7 @@ class JobRunner:
|
||||
if job._stop_event.is_set():
|
||||
job.status = JobStatus.STOPPED
|
||||
logger.warning("Job was stopped during execution")
|
||||
self._discard_generators()
|
||||
else:
|
||||
job.status = JobStatus.COMPLETED
|
||||
buf.progress = 100.0
|
||||
@@ -1156,9 +1504,19 @@ class JobRunner:
|
||||
self._save_job(job)
|
||||
buf.phase = "done"
|
||||
|
||||
except UnresolvedReference as exception:
|
||||
logger.error("Job %s can't get its reference frame: %s", job.id, exception)
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = str(exception)
|
||||
job.finished_at = time.time()
|
||||
self._save_job(job)
|
||||
buf.phase = "failed"
|
||||
|
||||
except Exception as exception:
|
||||
error_msg = str(exception)
|
||||
logger.exception("Critical error in job thread: %s", error_msg)
|
||||
# The model may be mid-request. Reload rather than trust it for the next job.
|
||||
self._discard_generators()
|
||||
job.status = JobStatus.FAILED
|
||||
job.error = f"Critical error ({type(exception).__name__}): {error_msg}"
|
||||
job.finished_at = time.time()
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Join a scene's clips, in order, into one video.
|
||||
|
||||
Kept apart from ``server.py`` because it needs nothing from FastVideo -- only
|
||||
PyAV (to read each clip's format) and an ``ffmpeg`` binary -- so it can be
|
||||
exercised without a GPU.
|
||||
|
||||
Clips from one scene share codec, size and frame rate, so they are joined by
|
||||
copying the streams rather than re-encoding: it takes about a second, loses no
|
||||
quality, and keeps picture and sound in step. Each clip is cut off at the end of
|
||||
its last video frame, because the audio track runs a few milliseconds past it and
|
||||
would otherwise open a gap between clips.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
VIDEO_EXTENSIONS = {".mp4", ".mov", ".mkv", ".webm", ".avi"}
|
||||
MERGED_DIRNAME = "merged"
|
||||
_FILENAME_RE = re.compile(r"^[\w.-]+\.mp4$")
|
||||
|
||||
|
||||
class MergeError(Exception):
|
||||
"""A merge that can't be done, carrying the HTTP status to report."""
|
||||
|
||||
def __init__(self, status_code: int, detail: str):
|
||||
super().__init__(detail)
|
||||
self.status_code = status_code
|
||||
self.detail = detail
|
||||
|
||||
|
||||
def find_ffmpeg() -> str | None:
|
||||
"""An ``ffmpeg`` to run: $FASTVIDEO_FFMPEG_BIN, then PATH, then imageio-ffmpeg's bundled one."""
|
||||
configured = os.getenv("FASTVIDEO_FFMPEG_BIN")
|
||||
if configured and (shutil.which(configured) or os.path.isfile(configured)):
|
||||
return shutil.which(configured) or configured
|
||||
on_path = shutil.which("ffmpeg")
|
||||
if on_path:
|
||||
return on_path
|
||||
try:
|
||||
import imageio_ffmpeg
|
||||
|
||||
return imageio_ffmpeg.get_ffmpeg_exe()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ClipInfo:
|
||||
path: str
|
||||
video_codec: str
|
||||
width: int
|
||||
height: int
|
||||
fps: float
|
||||
pix_fmt: str
|
||||
video_seconds: float
|
||||
audio: tuple[str, int, int] | None # codec, sample rate, channels
|
||||
|
||||
def describe(self) -> str:
|
||||
audio = f"{self.audio[0]} {self.audio[1]} Hz x{self.audio[2]}" if self.audio else "no audio"
|
||||
return f"{self.width}x{self.height} {self.fps:.2f} fps {self.video_codec}/{self.pix_fmt}, {audio}"
|
||||
|
||||
def signature(self) -> tuple[Any, ...]:
|
||||
return (self.video_codec, self.width, self.height, round(self.fps, 3), self.pix_fmt, self.audio)
|
||||
|
||||
|
||||
def probe_clip(path: str) -> ClipInfo:
|
||||
"""Read what ``check_compatible`` needs from a video file."""
|
||||
import av
|
||||
|
||||
try:
|
||||
with av.open(path) as container:
|
||||
video = next((s for s in container.streams if s.type == "video"), None)
|
||||
if video is None:
|
||||
raise MergeError(409, f"{os.path.basename(path)} has no video stream.")
|
||||
audio = next((s for s in container.streams if s.type == "audio"), None)
|
||||
fps = float(video.average_rate or video.guessed_rate or 0)
|
||||
if video.duration is not None:
|
||||
seconds = float(video.duration * video.time_base)
|
||||
elif container.duration is not None:
|
||||
seconds = container.duration / 1_000_000
|
||||
else:
|
||||
seconds = (video.frames or 0) / fps if fps else 0.0
|
||||
if seconds <= 0:
|
||||
raise MergeError(409, f"{os.path.basename(path)} has no playable video.")
|
||||
return ClipInfo(
|
||||
path=path,
|
||||
video_codec=video.codec_context.name,
|
||||
width=video.codec_context.width,
|
||||
height=video.codec_context.height,
|
||||
fps=fps,
|
||||
pix_fmt=video.codec_context.pix_fmt or "",
|
||||
video_seconds=seconds,
|
||||
audio=((audio.codec_context.name, audio.codec_context.sample_rate, audio.codec_context.channels)
|
||||
if audio else None),
|
||||
)
|
||||
except MergeError:
|
||||
raise
|
||||
except Exception as exc: # av.error.* on a corrupt or unreadable file
|
||||
raise MergeError(409, f"Could not read {os.path.basename(path)}: {exc}") from exc
|
||||
|
||||
|
||||
def check_compatible(infos: list[ClipInfo], labels: list[str]) -> None:
|
||||
"""Refuse clips that can't be joined by copying, naming the first that differs."""
|
||||
first = infos[0]
|
||||
for info, label in zip(infos[1:], labels[1:], strict=True):
|
||||
if info.signature() != first.signature():
|
||||
raise MergeError(
|
||||
409,
|
||||
f"{label} ({info.describe()}) doesn't match {labels[0]} ({first.describe()}). "
|
||||
"Clips need the same size, frame rate and audio to be joined.",
|
||||
)
|
||||
|
||||
|
||||
def _concat_list(infos: list[ClipInfo]) -> str:
|
||||
lines = ["ffconcat version 1.0"]
|
||||
for info in infos:
|
||||
escaped = info.path.replace("'", "'\\''")
|
||||
lines += [f"file '{escaped}'", f"outpoint {info.video_seconds:.6f}"]
|
||||
return "\n".join(lines) + "\n"
|
||||
|
||||
|
||||
def merge_clips(paths: list[str], out_path: str, labels: list[str] | None = None, ffmpeg: str | None = None) -> float:
|
||||
"""Join ``paths`` in order into ``out_path``. Returns the merged length in seconds."""
|
||||
if not paths:
|
||||
raise MergeError(400, "No clips to merge.")
|
||||
labels = labels or [os.path.basename(p) for p in paths]
|
||||
ffmpeg = ffmpeg or find_ffmpeg()
|
||||
if not ffmpeg:
|
||||
raise MergeError(503, "ffmpeg isn't available on the server. Install it or set FASTVIDEO_FFMPEG_BIN.")
|
||||
|
||||
infos = [probe_clip(p) for p in paths]
|
||||
check_compatible(infos, labels)
|
||||
|
||||
os.makedirs(os.path.dirname(out_path) or ".", exist_ok=True)
|
||||
tmp_out = f"{out_path}.tmp.mp4"
|
||||
list_fd, list_path = tempfile.mkstemp(suffix=".ffconcat", text=True)
|
||||
try:
|
||||
with os.fdopen(list_fd, "w", encoding="utf-8") as f:
|
||||
f.write(_concat_list(infos))
|
||||
result = subprocess.run(
|
||||
[ffmpeg, "-y", "-loglevel", "error", "-f", "concat", "-safe", "0", "-i", list_path, "-c", "copy",
|
||||
"-movflags", "+faststart", tmp_out],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
check=False,
|
||||
)
|
||||
if result.returncode != 0:
|
||||
tail = " ".join(result.stderr.strip().splitlines()[-3:])
|
||||
raise MergeError(500, f"ffmpeg failed: {tail or 'unknown error'}")
|
||||
os.replace(tmp_out, out_path) # atomic, so a reader never sees a half-written file
|
||||
except subprocess.TimeoutExpired as exc:
|
||||
raise MergeError(500, "ffmpeg took too long.") from exc
|
||||
finally:
|
||||
for leftover in (list_path, tmp_out):
|
||||
try:
|
||||
os.remove(leftover)
|
||||
except OSError:
|
||||
pass
|
||||
return sum(i.video_seconds for i in infos)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class MergeResult:
|
||||
filename: str
|
||||
path: str
|
||||
clips: int
|
||||
seconds: float
|
||||
|
||||
|
||||
def _safe_stem(name: str) -> str:
|
||||
return re.sub(r"[^\w.-]+", "-", name).strip("-.")[:60] or "scene"
|
||||
|
||||
|
||||
def _label(job: Any) -> str:
|
||||
name = (getattr(job, "name", "") or "").strip()
|
||||
return f"'{name}'" if name else str(job.id)
|
||||
|
||||
|
||||
def merge_jobs(jobs: list[Any], out_dir: str, name: str = "") -> MergeResult:
|
||||
"""Join the finished videos of ``jobs`` (in the order given) into ``<out_dir>/merged``.
|
||||
|
||||
Every job must be completed with a video on disk; otherwise nothing is made,
|
||||
so a scene missing a clip can't be mistaken for the whole thing.
|
||||
"""
|
||||
if not jobs:
|
||||
raise MergeError(400, "No clips to merge.")
|
||||
paths: list[str] = []
|
||||
for job in jobs:
|
||||
status = getattr(job.status, "value", job.status)
|
||||
if status != "completed" or not job.output_path:
|
||||
raise MergeError(409, f"{_label(job)} hasn't finished ({status}), so the scene can't be merged yet.")
|
||||
if not os.path.isfile(job.output_path):
|
||||
raise MergeError(409, f"{_label(job)}'s video is missing from disk.")
|
||||
if Path(job.output_path).suffix.lower() not in VIDEO_EXTENSIONS:
|
||||
raise MergeError(409, f"{_label(job)}'s output isn't a video.")
|
||||
paths.append(job.output_path)
|
||||
|
||||
filename = f"{_safe_stem(name)}-{time.strftime('%Y%m%d-%H%M%S')}.mp4"
|
||||
merged_dir = os.path.join(out_dir, MERGED_DIRNAME)
|
||||
out_path = os.path.join(merged_dir, filename)
|
||||
seconds = merge_clips(paths, out_path, labels=[_label(j) for j in jobs])
|
||||
return MergeResult(filename=filename, path=out_path, clips=len(paths), seconds=seconds)
|
||||
|
||||
|
||||
def merged_path(out_dir: str, filename: str) -> str:
|
||||
"""Path of a merged video by filename, refusing anything that isn't one."""
|
||||
if not _FILENAME_RE.match(filename):
|
||||
raise MergeError(404, "Merged video not found.")
|
||||
path = os.path.join(out_dir, MERGED_DIRNAME, filename)
|
||||
if not os.path.isfile(path):
|
||||
raise MergeError(404, "Merged video not found.")
|
||||
return path
|
||||
@@ -4,8 +4,8 @@ In-memory mock of the FastVideo Studio API (``server.py``).
|
||||
|
||||
This mock implements the same ``/api`` routes and response shapes as the real
|
||||
FastAPI server but keeps everything in memory and never touches the FastVideo
|
||||
library, a GPU, or a database. It is meant to back the Playwright e2e suite so
|
||||
the Next.js frontend can be exercised end-to-end without a real backend.
|
||||
library or a GPU. It is meant to back the Playwright e2e suite so the Next.js
|
||||
frontend can be exercised end-to-end without a real backend.
|
||||
|
||||
Job lifecycle is simulated by recording a start timestamp and *computing* the
|
||||
status on read: a started job reports ``running`` for a few seconds and then
|
||||
@@ -13,14 +13,32 @@ flips to ``completed`` with an ``output_path``, so polling the job list / logs
|
||||
shows progression. Generated media is a tiny 1s ``testsrc`` MP4 built lazily
|
||||
with ffmpeg and cached.
|
||||
|
||||
Pass ``--data-dir`` to seed jobs/datasets/settings from a real server's
|
||||
persisted SQLite database instead of the built-in fake demo data, so you can
|
||||
browse real job history through the mock without a GPU. This is a read-only
|
||||
snapshot taken once at startup: nothing created or changed while the mock
|
||||
runs is ever written back to that database, so it's safe to point at the
|
||||
same ``--data-dir`` a real ``server.py`` is using.
|
||||
|
||||
Pass ``--state-file`` to keep the mock's own job changes across restarts:
|
||||
after every job create/edit/queue/delete the mock writes the jobs it created
|
||||
or changed (and the snapshot jobs it deleted) to that JSON file, and applies
|
||||
it on top of the snapshot at the next startup. The SQLite database is still
|
||||
never written; delete the file to reset the mock.
|
||||
|
||||
Usage (from the ``apps/`` directory)::
|
||||
|
||||
PYTHONPATH=.. python -m fastvideo_studio.mock_server --port 8190
|
||||
PYTHONPATH=.. python -m fastvideo_studio.mock_server --port 8190 --data-dir ../outputs/ui_data \
|
||||
--state-file ../outputs/ui_data/mock_state.json
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
@@ -29,15 +47,26 @@ import tempfile
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from types import SimpleNamespace
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import FastAPI, File, HTTPException, UploadFile
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import FileResponse, PlainTextResponse
|
||||
|
||||
from fastvideo_studio.database import default_settings_dict
|
||||
from fastvideo_studio.models import (CreateDatasetRequest, CreateJobRequest, SettingsUpdate, UpdateCaptionRequest,
|
||||
model_label)
|
||||
from fastvideo_studio.database import Database, default_settings_dict
|
||||
from fastvideo_studio.job_queue import circular_dependency, dependencies, plan_dispatch
|
||||
from fastvideo_studio.merge import MergeError, merge_jobs, merged_path
|
||||
from fastvideo_studio.reference_preview import reference_image_response
|
||||
from fastvideo_studio.scene_writer.routes import router as scene_writer_router
|
||||
from fastvideo_studio.scene_writer.runner import use_server_data_dir
|
||||
from fastvideo_studio.trim import BRIGHTNESS_RANGE, CONTRAST_RANGE, SATURATION_RANGE
|
||||
from fastvideo_studio.models import (CreateDatasetRequest, CreateJobRequest, GradeSegment, MergeSceneRequest,
|
||||
QueueJobsRequest, SettingsUpdate, TrimRequest, UpdateCaptionRequest, model_label)
|
||||
|
||||
logging.basicConfig(level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s")
|
||||
logger = logging.getLogger("fastvideo.studio.mock")
|
||||
|
||||
# --- Config -----------------------------------------------------------------
|
||||
|
||||
@@ -45,6 +74,11 @@ from fastvideo_studio.models import (CreateDatasetRequest, CreateJobRequest, Set
|
||||
COMPLETE_AFTER_SECONDS = 3.0
|
||||
FFMPEG_BIN = shutil.which(os.getenv("FASTVIDEO_FFMPEG_BIN", "ffmpeg"))
|
||||
|
||||
# Marks a job dict as a frozen row loaded from a real --data-dir snapshot
|
||||
# (see _load_snapshot_from_db) rather than something this process is
|
||||
# simulating. Stripped from API responses by _public_job.
|
||||
_FROM_SNAPSHOT_KEY = "_from_db_snapshot"
|
||||
|
||||
# A small catalogue of fake models keyed by workload type. Mirrors the real
|
||||
# server's {id, label} shape (label derived from the HF-style path).
|
||||
_MODELS_BY_WORKLOAD: dict[str, list[str]] = {
|
||||
@@ -54,6 +88,10 @@ _MODELS_BY_WORKLOAD: dict[str, list[str]] = {
|
||||
],
|
||||
"i2v": [
|
||||
"Wan-AI/Wan2.1-I2V-14B-480P-Diffusers",
|
||||
# Real registered id (fastvideo/registry.py) -- needed so the frontend's
|
||||
# `modelId.toLowerCase().includes('minimax-h3')` gate (speech-tag
|
||||
# console, references, last-frame upload) actually lights up in the mock.
|
||||
"MiniMaxAI/MiniMax-H3",
|
||||
],
|
||||
"t2i": [
|
||||
"black-forest-labs/FLUX.1-schnell",
|
||||
@@ -91,6 +129,9 @@ _jobs: dict[str, dict[str, Any]] = {}
|
||||
_datasets: dict[str, dict[str, Any]] = {}
|
||||
# dataset_id -> {"file_names": [...], "captions": {file_name: caption}}
|
||||
_dataset_files: dict[str, dict[str, Any]] = {}
|
||||
# Each snapshot job as loaded, so --state-file saves only what the mock changed.
|
||||
_snapshot_originals: dict[str, dict[str, Any]] = {}
|
||||
_state_file: Path | None = None
|
||||
|
||||
# Lazily-built, cached media clips (shared by every job/dataset media response),
|
||||
# keyed by extension: "mp4" (1s testsrc video) or "png" (single testsrc frame).
|
||||
@@ -98,16 +139,77 @@ _mock_media_cache: dict[str, str] = {}
|
||||
_mock_media_lock = threading.Lock()
|
||||
|
||||
|
||||
def _testsrc_frame(width: int, height: int, index: int):
|
||||
"""A cheap, dependency-light stand-in for ffmpeg's `testsrc` pattern:
|
||||
a gradient that shifts over `index` so an encoded clip visibly animates."""
|
||||
import numpy as np
|
||||
|
||||
frame = np.zeros((height, width, 3), dtype=np.uint8)
|
||||
frame[:, :, 0] = np.linspace(0, 255, width, dtype=np.uint8)[None, :]
|
||||
frame[:, :, 1] = np.linspace(0, 255, height, dtype=np.uint8)[:, None]
|
||||
frame[:, :, 2] = (index * 10) % 256
|
||||
return frame
|
||||
|
||||
|
||||
def _build_mock_media_with_pyav(kind: str, path: str) -> None:
|
||||
"""ffmpeg isn't installed on every dev box; PyAV (a real FastVideo
|
||||
dependency already, used for reference-video decoding) can encode the
|
||||
same tiny clip directly via its bundled libav, no external binary."""
|
||||
from PIL import Image
|
||||
|
||||
if kind == "png":
|
||||
Image.fromarray(_testsrc_frame(320, 240, 0)).save(path)
|
||||
return
|
||||
|
||||
import av as pyav
|
||||
|
||||
# h264 is what every browser's <video> element actually plays; mpeg4
|
||||
# (MPEG-4 Part 2) is a PyAV-always-has-it fallback that no mainstream
|
||||
# browser decodes, so gallery/job previews would just fail to load.
|
||||
try:
|
||||
pyav.codec.Codec("h264", "w")
|
||||
codec_name = "h264"
|
||||
except Exception:
|
||||
logger.warning("PyAV has no h264 encoder available; falling back to mpeg4, which most "
|
||||
"browsers cannot play in <video>. Install ffmpeg for real preview playback.")
|
||||
codec_name = "mpeg4"
|
||||
|
||||
container = pyav.open(path, mode="w")
|
||||
try:
|
||||
stream = container.add_stream(codec_name, rate=24)
|
||||
stream.width = 320
|
||||
stream.height = 240
|
||||
stream.pix_fmt = "yuv420p"
|
||||
for i in range(24): # 1s at 24fps, matching the ffmpeg path's clip length
|
||||
frame = pyav.VideoFrame.from_ndarray(_testsrc_frame(320, 240, i), format="rgb24")
|
||||
for packet in stream.encode(frame):
|
||||
container.mux(packet)
|
||||
for packet in stream.encode():
|
||||
container.mux(packet)
|
||||
finally:
|
||||
container.close()
|
||||
|
||||
|
||||
def _build_mock_media(kind: str) -> str:
|
||||
"""Build (once) a tiny testsrc clip of the given ``kind`` with ffmpeg; cache the path."""
|
||||
"""Build (once) a tiny testsrc-like clip of the given ``kind``; cache the path."""
|
||||
with _mock_media_lock:
|
||||
cached = _mock_media_cache.get(kind)
|
||||
if cached and os.path.isfile(cached):
|
||||
return cached
|
||||
if not FFMPEG_BIN:
|
||||
raise HTTPException(status_code=500, detail="ffmpeg is required to build mock media")
|
||||
fd, path = tempfile.mkstemp(prefix="fvstudio_mock_", suffix=f".{kind}")
|
||||
os.close(fd)
|
||||
if not FFMPEG_BIN:
|
||||
try:
|
||||
_build_mock_media_with_pyav(kind, path)
|
||||
except Exception as exc:
|
||||
with contextlib.suppress(OSError):
|
||||
os.remove(path)
|
||||
raise HTTPException(status_code=500,
|
||||
detail=f"PyAV fallback failed to build mock {kind}: {exc}") from exc
|
||||
if not os.path.isfile(path) or os.path.getsize(path) == 0:
|
||||
raise HTTPException(status_code=500, detail=f"PyAV fallback produced no mock {kind} bytes")
|
||||
_mock_media_cache[kind] = path
|
||||
return path
|
||||
if kind == "png":
|
||||
command = [
|
||||
FFMPEG_BIN,
|
||||
@@ -166,33 +268,34 @@ def _build_mock_media(kind: str) -> str:
|
||||
|
||||
|
||||
def _new_job_dict(req: CreateJobRequest) -> dict[str, Any]:
|
||||
"""Build a job dict (mirrors job_runner.Job.to_dict()) in the pending state."""
|
||||
"""Build a job dict (mirrors job_runner.Job.to_dict()) in the pending state.
|
||||
|
||||
Starts from req.model_dump() rather than re-listing CreateJobRequest's
|
||||
fields by hand -- the previous version silently dropped name,
|
||||
last_image_path, references, and every offload/tuning flag because it
|
||||
was never updated as CreateJobRequest grew those fields. Spreading the
|
||||
request means new fields flow through automatically.
|
||||
"""
|
||||
job_id = str(uuid.uuid4())
|
||||
return {
|
||||
**req.model_dump(),
|
||||
"id": job_id,
|
||||
"model_id": req.model_id,
|
||||
"prompt": req.prompt,
|
||||
"workload_type": req.workload_type or "t2v",
|
||||
"job_type": req.job_type or "inference",
|
||||
"image_path": req.image_path or "",
|
||||
"negative_prompt": req.negative_prompt or "",
|
||||
"data_path": req.data_path or "",
|
||||
"status": "pending",
|
||||
"created_at": time.time(),
|
||||
"queued_at": None,
|
||||
"started_at": None,
|
||||
"finished_at": None,
|
||||
"error": None,
|
||||
"output_path": None,
|
||||
"log_file_path": None,
|
||||
"num_inference_steps": req.num_inference_steps,
|
||||
"num_frames": req.num_frames,
|
||||
"height": req.height,
|
||||
"width": req.width,
|
||||
"guidance_scale": req.guidance_scale,
|
||||
"guidance_rescale": req.guidance_rescale,
|
||||
"fps": req.fps,
|
||||
"seed": req.seed,
|
||||
"negative_prompt": req.negative_prompt or "",
|
||||
"num_gpus": req.num_gpus,
|
||||
"data_path": req.data_path or "",
|
||||
"edit_start_seconds": 0.0,
|
||||
"edit_end_seconds": None,
|
||||
"edit_segments": [],
|
||||
"progress": 0.0,
|
||||
"progress_msg": "",
|
||||
"phase": "pending",
|
||||
@@ -204,8 +307,13 @@ def _advance_job(job: dict[str, Any]) -> None:
|
||||
|
||||
Status is *computed on read* from the recorded start timestamp, so polling
|
||||
the job list / logs naturally shows pending -> running -> completed.
|
||||
|
||||
Jobs loaded from a persisted DB snapshot (``_FROM_SNAPSHOT_KEY``) are a
|
||||
frozen historical record, not something this process is simulating — skip
|
||||
them so a real "pending" or "running" row doesn't silently start ticking
|
||||
toward a fake "completed" just because someone polled it.
|
||||
"""
|
||||
if job["status"] != "running" or not job.get("started_at"):
|
||||
if job.get(_FROM_SNAPSHOT_KEY) or job["status"] != "running" or not job.get("started_at"):
|
||||
return
|
||||
elapsed = time.time() - job["started_at"]
|
||||
if elapsed >= COMPLETE_AFTER_SECONDS:
|
||||
@@ -219,9 +327,9 @@ def _advance_job(job: dict[str, Any]) -> None:
|
||||
|
||||
|
||||
def _public_job(job: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Advance + return a copy safe to serialize."""
|
||||
"""Advance + return a copy safe to serialize (internal-only keys stripped)."""
|
||||
_advance_job(job)
|
||||
return dict(job)
|
||||
return {k: v for k, v in job.items() if not k.startswith("_")}
|
||||
|
||||
|
||||
_LOG_TAIL = [
|
||||
@@ -252,8 +360,8 @@ def _compute_logs(job: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Return JobLogs-shaped data, growing the visible lines as time passes."""
|
||||
seq = _log_sequence(job)
|
||||
status = job["status"]
|
||||
if status == "pending":
|
||||
return {"lines": [], "progress": 0.0, "progress_msg": "", "phase": "pending"}
|
||||
if status in ("pending", "queued"):
|
||||
return {"lines": [], "progress": 0.0, "progress_msg": "", "phase": status}
|
||||
if status == "completed":
|
||||
return {"lines": seq, "progress": 100.0, "progress_msg": "50/50 steps", "phase": "done"}
|
||||
if status in ("failed", "stopped"):
|
||||
@@ -346,6 +454,131 @@ def _seed() -> None:
|
||||
}
|
||||
|
||||
|
||||
# --- DB snapshot (--data-dir) ------------------------------------------------
|
||||
|
||||
|
||||
def _decode_json_list(value: Any) -> list[dict[str, Any]]:
|
||||
"""References and edit_segments both round-trip through the DB as JSON text."""
|
||||
if not value:
|
||||
return []
|
||||
if isinstance(value, list):
|
||||
return list(value)
|
||||
try:
|
||||
decoded = json.loads(value)
|
||||
except (TypeError, ValueError):
|
||||
return []
|
||||
return list(decoded) if isinstance(decoded, list) else []
|
||||
|
||||
|
||||
def _job_from_snapshot(row: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Adapt a persisted job row into the mock's job dict shape.
|
||||
|
||||
Frozen as a read-only historical record (see _FROM_SNAPSHOT_KEY): the mock
|
||||
never actually ran this job, so it doesn't get to simulate progress for
|
||||
one. A row still marked "running" is orphaned -- the real JobRunner
|
||||
reclassifies those as failed on its own restart, so mirror that here.
|
||||
"""
|
||||
job = dict(row)
|
||||
status = job["status"]
|
||||
if status == "running":
|
||||
status = "failed"
|
||||
job["error"] = job.get("error") or "Server restarted (job was running)"
|
||||
job["finished_at"] = job.get("finished_at") or time.time()
|
||||
elif status == "queued":
|
||||
# The real server resumes its queue on restart; the mock never runs
|
||||
# snapshot jobs, so leave it as something the user can queue again.
|
||||
status = "pending"
|
||||
job["queued_at"] = None
|
||||
job["status"] = status
|
||||
job["references"] = _decode_json_list(row.get("references"))
|
||||
job["edit_segments"] = _decode_json_list(row.get("edit_segments"))
|
||||
job["progress"] = 100.0 if status == "completed" else 0.0
|
||||
job["progress_msg"] = "50/50 steps" if status == "completed" else ""
|
||||
job["phase"] = "done" if status == "completed" else status
|
||||
job[_FROM_SNAPSHOT_KEY] = True
|
||||
return job
|
||||
|
||||
|
||||
def _load_snapshot_from_db(data_dir: Path) -> bool:
|
||||
"""Seed jobs/datasets/settings from a real server's persisted SQLite db.
|
||||
|
||||
Read-only: opens the db (Database.__init__ runs the same idempotent
|
||||
schema migrations a real server startup would) and copies rows into the
|
||||
mock's in-memory state; nothing the mock does afterward is written back.
|
||||
Returns False (leaving the caller to fall back to _seed()) if no db file
|
||||
is found at that path, rather than silently creating an empty one.
|
||||
"""
|
||||
db_path = data_dir / "fastvideo_ui.db"
|
||||
if not db_path.is_file():
|
||||
logger.warning("--data-dir %s has no fastvideo_ui.db; falling back to fake demo data.", data_dir)
|
||||
return False
|
||||
|
||||
db = Database(db_path)
|
||||
settings = db.get_settings()
|
||||
jobs = [_job_from_snapshot(row) for row in db.get_all_jobs()]
|
||||
datasets = db.get_all_datasets()
|
||||
|
||||
with _state_lock:
|
||||
_settings.update({k: v for k, v in settings.items() if v is not None})
|
||||
_jobs.clear()
|
||||
_jobs.update({j["id"]: j for j in jobs})
|
||||
_snapshot_originals.clear()
|
||||
_snapshot_originals.update({j["id"]: copy.deepcopy(j) for j in jobs})
|
||||
_datasets.clear()
|
||||
_datasets.update({d["id"]: d for d in datasets})
|
||||
_dataset_files.clear()
|
||||
for d in datasets:
|
||||
captions = db.get_dataset_captions(d["id"])
|
||||
_dataset_files[d["id"]] = {"file_names": list(captions.keys()), "captions": captions}
|
||||
|
||||
logger.info(
|
||||
"Loaded read-only snapshot from %s: %d job(s), %d dataset(s).",
|
||||
db_path,
|
||||
len(jobs),
|
||||
len(datasets),
|
||||
)
|
||||
return True
|
||||
|
||||
|
||||
# --- Saved mock state (--state-file) -----------------------------------------
|
||||
|
||||
|
||||
def _save_state() -> None:
|
||||
"""Write the jobs the mock created or changed to _state_file. Lock held.
|
||||
|
||||
Snapshot jobs left as loaded are not saved, so later changes to the real
|
||||
database still show through; deleted snapshot jobs are listed by id.
|
||||
"""
|
||||
if _state_file is None:
|
||||
return
|
||||
jobs = [j for jid, j in _jobs.items() if _snapshot_originals.get(jid) != j]
|
||||
deleted = [jid for jid in _snapshot_originals if jid not in _jobs]
|
||||
tmp = _state_file.with_name(_state_file.name + ".tmp")
|
||||
try:
|
||||
tmp.write_text(json.dumps({"jobs": jobs, "deleted_job_ids": deleted}, indent=1))
|
||||
os.replace(tmp, _state_file)
|
||||
except (OSError, TypeError, ValueError):
|
||||
logger.exception("Could not save mock state to %s", _state_file)
|
||||
|
||||
|
||||
def _load_state(path: Path) -> None:
|
||||
"""Apply a saved --state-file on top of whatever was seeded or snapshotted."""
|
||||
if not path.is_file():
|
||||
logger.info("No mock state at %s yet; it will be created on the first job change.", path)
|
||||
return
|
||||
try:
|
||||
state = json.loads(path.read_text())
|
||||
except (OSError, ValueError):
|
||||
logger.exception("Could not read mock state from %s; starting without it.", path)
|
||||
return
|
||||
with _state_lock:
|
||||
for job_id in state.get("deleted_job_ids", []):
|
||||
_jobs.pop(job_id, None)
|
||||
for job in state.get("jobs", []):
|
||||
_jobs[job["id"]] = job
|
||||
logger.info("Loaded %d saved job(s) from %s.", len(state.get("jobs", [])), path)
|
||||
|
||||
|
||||
# --- App --------------------------------------------------------------------
|
||||
|
||||
app = FastAPI(title="FastVideo Studio Mock API", version="0.1.0")
|
||||
@@ -356,8 +589,18 @@ app.add_middleware(
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
# The real scene writer: drafting runs the agent against a real model, but
|
||||
# writes only job files and creates nothing, so it is safe beside mock jobs.
|
||||
app.include_router(scene_writer_router)
|
||||
|
||||
_seed()
|
||||
|
||||
@app.middleware("http")
|
||||
async def save_state_after_job_changes(request, call_next):
|
||||
response = await call_next(request)
|
||||
if _state_file is not None and request.method != "GET" and request.url.path.startswith("/api/jobs"):
|
||||
with _state_lock:
|
||||
_save_state()
|
||||
return response
|
||||
|
||||
|
||||
@app.get("/api/__mock__")
|
||||
@@ -423,6 +666,32 @@ async def upload_image(file: Annotated[UploadFile, File()]) -> dict[str, str]:
|
||||
return {"path": f"/mock/uploads/{uuid.uuid4().hex}_{os.path.basename(name)}"}
|
||||
|
||||
|
||||
ALLOWED_REFERENCE_VIDEO_EXTENSIONS = {".mp4", ".mov", ".mkv", ".webm", ".avi"}
|
||||
ALLOWED_REFERENCE_AUDIO_EXTENSIONS = {".wav", ".mp3", ".flac", ".m4a", ".ogg"}
|
||||
ALLOWED_REFERENCE_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".bmp"}
|
||||
|
||||
|
||||
@app.post("/api/upload-media")
|
||||
async def upload_media(file: Annotated[UploadFile, File()]) -> dict[str, str]:
|
||||
"""Ref2VA reference upload (image/video/audio) -- the real server's
|
||||
counterpart to upload_image, which only ever handles images."""
|
||||
name = file.filename or "reference"
|
||||
ext = os.path.splitext(name)[1].lower()
|
||||
allowed = ALLOWED_REFERENCE_IMAGE_EXTENSIONS | ALLOWED_REFERENCE_VIDEO_EXTENSIONS | ALLOWED_REFERENCE_AUDIO_EXTENSIONS
|
||||
if ext not in allowed:
|
||||
raise HTTPException(status_code=400, detail=f"Invalid file type. Allowed: {', '.join(sorted(allowed))}")
|
||||
if ext in ALLOWED_REFERENCE_VIDEO_EXTENSIONS:
|
||||
media_type = "video"
|
||||
elif ext in ALLOWED_REFERENCE_AUDIO_EXTENSIONS:
|
||||
media_type = "audio"
|
||||
else:
|
||||
media_type = "image"
|
||||
return {
|
||||
"path": f"/mock/uploads/{uuid.uuid4().hex}_{os.path.basename(name)}",
|
||||
"media_type": media_type,
|
||||
}
|
||||
|
||||
|
||||
@app.post("/api/upload-raw-dataset")
|
||||
async def upload_raw_dataset(files: Annotated[list[UploadFile], File()]) -> dict[str, Any]:
|
||||
video_exts = {".mp4", ".webm", ".avi", ".mov", ".mkv"}
|
||||
@@ -446,6 +715,7 @@ def list_jobs(job_type: str | None = None) -> list[dict[str, Any]]:
|
||||
# _public_job mutates the shared job dict via _advance_job, so it must run
|
||||
# under the lock (matching every other job route) to avoid racing start/stop.
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
jobs = list(_jobs.values())
|
||||
if job_type:
|
||||
jobs = [j for j in jobs if j.get("job_type") == job_type]
|
||||
@@ -456,6 +726,7 @@ def list_jobs(job_type: str | None = None) -> list[dict[str, Any]]:
|
||||
@app.get("/api/jobs/{job_id}")
|
||||
def get_job(job_id: str) -> dict[str, Any]:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
@@ -466,15 +737,31 @@ def get_job(job_id: str) -> dict[str, Any]:
|
||||
def create_job(req: CreateJobRequest) -> dict[str, Any]:
|
||||
job = _new_job_dict(req)
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
_jobs[job["id"]] = job
|
||||
if _settings.get("autoStartJob"):
|
||||
_start(job)
|
||||
return _public_job(job)
|
||||
|
||||
|
||||
def _start(job: dict[str, Any]) -> None:
|
||||
@app.post("/api/jobs/{job_id}/duplicate", status_code=201)
|
||||
def duplicate_job(job_id: str) -> dict[str, Any]:
|
||||
"""A new pending job with an existing job's configuration, like the real server's."""
|
||||
with _state_lock:
|
||||
source = _jobs.get(job_id)
|
||||
if source is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
config = {k: copy.deepcopy(source[k]) for k in CreateJobRequest.model_fields if k in source}
|
||||
return create_job(CreateJobRequest(**config))
|
||||
|
||||
|
||||
def _start(job: dict[str, Any], at: float | None = None) -> None:
|
||||
# Once the mock is actually driving this job, it's no longer a frozen
|
||||
# historical record -- let _advance_job simulate it like any other.
|
||||
job.pop(_FROM_SNAPSHOT_KEY, None)
|
||||
job["status"] = "running"
|
||||
job["started_at"] = time.time()
|
||||
job["queued_at"] = None
|
||||
job["started_at"] = at if at is not None else time.time()
|
||||
job["finished_at"] = None
|
||||
job["error"] = None
|
||||
job["output_path"] = None
|
||||
@@ -486,17 +773,289 @@ def _start(job: dict[str, Any]) -> None:
|
||||
job["phase"] = "starting"
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/start")
|
||||
def start_job(job_id: str) -> dict[str, Any]:
|
||||
class _QueueView:
|
||||
"""The scheduler's view of a mock job dict (it reads attributes, not keys)."""
|
||||
|
||||
def __init__(self, job: dict[str, Any]):
|
||||
self.id = job["id"]
|
||||
self.name = job.get("name", "")
|
||||
self.status = job["status"]
|
||||
self.references = job.get("references")
|
||||
self.queued_at = job.get("queued_at")
|
||||
|
||||
|
||||
def _drain_queue() -> None:
|
||||
"""Run the queue up to now. Call with ``_state_lock`` held.
|
||||
|
||||
Like everything else in the mock this is computed on read: each pass
|
||||
finishes jobs whose time is up, then starts whatever the queue allows.
|
||||
A job takes over the slot at the moment the previous job *finished* (not at
|
||||
the moment someone happened to poll), so a queue that has been sitting for
|
||||
a minute shows every clip already done rather than advancing one per poll.
|
||||
Same rules as the real runner: one at a time, oldest first, and a clip that
|
||||
takes another's last frame waits for it -- see job_queue.plan_dispatch.
|
||||
"""
|
||||
while True:
|
||||
for job in _jobs.values():
|
||||
_advance_job(job)
|
||||
plan = plan_dispatch({jid: _QueueView(j) for jid, j in _jobs.items()}, max_concurrent=1)
|
||||
now = time.time()
|
||||
for job_id, reason in plan.fail.items():
|
||||
job = _jobs[job_id]
|
||||
job["status"] = "failed"
|
||||
job["error"] = reason
|
||||
job["finished_at"] = now
|
||||
job["queued_at"] = None
|
||||
job["phase"] = "failed"
|
||||
if not plan.start:
|
||||
if plan.fail:
|
||||
continue # a failure can free a slot or fail more jobs
|
||||
return
|
||||
for job_id in plan.start:
|
||||
job = _jobs[job_id]
|
||||
finished = [j["finished_at"] for j in _jobs.values() if j.get("finished_at")]
|
||||
# The slot came free when the last job finished, if that was after the job was queued.
|
||||
slot_free = max([job.get("queued_at") or 0.0, *[f for f in finished if f <= now]])
|
||||
_start(job, at=min(now, slot_free))
|
||||
|
||||
|
||||
def _check_dependencies_finished(job: dict[str, Any]) -> None:
|
||||
for dep_id in dependencies(job.get("references")):
|
||||
dep = _jobs.get(dep_id)
|
||||
if dep is None:
|
||||
raise HTTPException(status_code=409,
|
||||
detail="This job starts from the ending of a job that no longer exists.")
|
||||
if dep["status"] != "completed":
|
||||
label = f"'{dep['name']}'" if dep.get("name") else dep["id"]
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail=(f"This job starts from the ending of {label}, which hasn't finished "
|
||||
f"(it is {dep['status']}). Queue it to run after that job."),
|
||||
)
|
||||
|
||||
|
||||
# Mirrors JobRunner.EDITABLE_STATUSES: exactly the jobs that can still be started.
|
||||
_EDITABLE_STATUSES = ("pending", "failed", "stopped")
|
||||
|
||||
|
||||
@app.patch("/api/jobs/{job_id}")
|
||||
def update_job(job_id: str, updates: dict[str, Any]) -> dict[str, Any]:
|
||||
"""Edit a job's configuration; only jobs that have not produced a result."""
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
_advance_job(job)
|
||||
if job["status"] == "running":
|
||||
raise HTTPException(status_code=409, detail="Job is already running")
|
||||
if job["status"] == "completed":
|
||||
raise HTTPException(status_code=409, detail="Job already completed. Delete and re-create to run again.")
|
||||
if job["status"] not in _EDITABLE_STATUSES:
|
||||
raise HTTPException(
|
||||
status_code=400,
|
||||
detail=(f"Job is {job['status']}; only {', '.join(_EDITABLE_STATUSES)} jobs can be edited. "
|
||||
"Duplicate it instead."),
|
||||
)
|
||||
unknown = set(updates) - set(CreateJobRequest.model_fields)
|
||||
if unknown:
|
||||
raise HTTPException(status_code=400, detail=f"Not editable: {', '.join(sorted(unknown))}")
|
||||
job.update(updates)
|
||||
return _public_job(job)
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/last-frame")
|
||||
def extract_last_frame(job_id: str) -> dict[str, str]:
|
||||
"""Stand-in for the real route: there is no video to read, so hand back a made-up path."""
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
_advance_job(job)
|
||||
if job["status"] != "completed" or not job.get("output_path"):
|
||||
raise HTTPException(status_code=404, detail="No output available for this job")
|
||||
return {"path": f"/mock/last_frames/last_frame_{job_id}.png", "media_type": "image"}
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/last-clip")
|
||||
def extract_last_clip(job_id: str) -> dict[str, str]:
|
||||
"""Stand-in for the real route: there is no video to read, so hand back a made-up path."""
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
_advance_job(job)
|
||||
if job["status"] != "completed" or not job.get("output_path"):
|
||||
raise HTTPException(status_code=404, detail="No output available for this job")
|
||||
return {"path": f"/mock/last_clips/last_clip_{job_id}.mp4", "media_type": "video"}
|
||||
|
||||
|
||||
#: Key holding a trimmed job's frame count before any trim, so it can be restored.
|
||||
_ORIGINAL_NUM_FRAMES_KEY = "_original_num_frames"
|
||||
|
||||
|
||||
def _trimmable_job(job_id: str) -> dict[str, Any]:
|
||||
"""Look up a completed job with an output, or raise. Call with _state_lock held."""
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
_advance_job(job)
|
||||
if job["status"] != "completed" or not job.get("output_path"):
|
||||
raise HTTPException(status_code=404, detail="No output available for this job")
|
||||
return job
|
||||
|
||||
|
||||
def _check_grade_range(name: str, value: float, bounds: tuple[float, float]) -> None:
|
||||
low, high = bounds
|
||||
if not (low <= value <= high):
|
||||
raise HTTPException(status_code=400, detail=f"{name} must be between {low:g} and {high:g}, got {value:g}.")
|
||||
|
||||
|
||||
def _check_segments(segments: list[GradeSegment]) -> None:
|
||||
"""Structural checks only (ordering, open-endedness, color bounds) -- the mock has no
|
||||
frames to count, so it doesn't reproduce trim.py's frame-rounding/coverage checks.
|
||||
|
||||
An empty list isn't an error here either, matching trim.py: it defaults to
|
||||
one neutral section covering everything (Pydantic's own default_factory
|
||||
already supplies that when the field is omitted, but an explicit "segments":
|
||||
[] reaches here as a real empty list).
|
||||
"""
|
||||
prev_end = 0.0
|
||||
for i, s in enumerate(segments):
|
||||
_check_grade_range("brightness", s.brightness, BRIGHTNESS_RANGE)
|
||||
_check_grade_range("contrast", s.contrast, CONTRAST_RANGE)
|
||||
_check_grade_range("saturation", s.saturation, SATURATION_RANGE)
|
||||
if s.end_seconds is None:
|
||||
if i != len(segments) - 1:
|
||||
raise HTTPException(status_code=400, detail="Only the last section can be open-ended.")
|
||||
else:
|
||||
if s.end_seconds <= prev_end:
|
||||
raise HTTPException(status_code=400, detail="Sections must be in increasing order.")
|
||||
prev_end = s.end_seconds
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/trim")
|
||||
def trim_job(job_id: str, req: TrimRequest) -> dict[str, Any]:
|
||||
"""Stand-in for the real route: no video to cut or grade, so just resize num_frames to match."""
|
||||
with _state_lock:
|
||||
job = _trimmable_job(job_id)
|
||||
if req.start_seconds < 0:
|
||||
raise HTTPException(status_code=400, detail="The start of the range can't be negative.")
|
||||
_check_segments(req.segments)
|
||||
fps = job.get("fps") or 24
|
||||
original_frames = job.setdefault(_ORIGINAL_NUM_FRAMES_KEY, job["num_frames"])
|
||||
duration = original_frames / fps
|
||||
end = duration if req.end_seconds is None else min(req.end_seconds, duration)
|
||||
if end <= req.start_seconds:
|
||||
raise HTTPException(status_code=400, detail="The end of the range must come after the start.")
|
||||
job["num_frames"] = max(1, round((end - req.start_seconds) * fps))
|
||||
# Recorded so the dialog can reopen showing what's actually applied,
|
||||
# like the real server's edit_* job fields.
|
||||
job["edit_start_seconds"] = req.start_seconds
|
||||
job["edit_end_seconds"] = req.end_seconds
|
||||
job["edit_segments"] = [s.model_dump() for s in req.segments]
|
||||
return _public_job(job)
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/restore-video")
|
||||
def restore_job_video(job_id: str) -> dict[str, Any]:
|
||||
"""Stand-in for the real route: undo the simulated edit by restoring num_frames."""
|
||||
with _state_lock:
|
||||
job = _trimmable_job(job_id)
|
||||
if _ORIGINAL_NUM_FRAMES_KEY not in job:
|
||||
raise HTTPException(status_code=404, detail="This job has not been edited.")
|
||||
job["num_frames"] = job.pop(_ORIGINAL_NUM_FRAMES_KEY)
|
||||
job["edit_start_seconds"] = 0.0
|
||||
job["edit_end_seconds"] = None
|
||||
job["edit_segments"] = []
|
||||
return _public_job(job)
|
||||
|
||||
|
||||
def _check_startable(job: dict[str, Any]) -> None:
|
||||
if job["status"] == "running":
|
||||
raise HTTPException(status_code=409, detail="Job is already running")
|
||||
if job["status"] == "queued":
|
||||
raise HTTPException(status_code=409, detail="Job is already queued")
|
||||
if job["status"] == "completed":
|
||||
raise HTTPException(status_code=409, detail="Job already completed. Delete and re-create to run again.")
|
||||
|
||||
|
||||
def _enqueue(job_ids: list[str]) -> list[dict[str, Any]]:
|
||||
"""Queue jobs in order, all-or-nothing (mirrors JobRunner.enqueue_jobs). Lock held."""
|
||||
if len(set(job_ids)) != len(job_ids):
|
||||
raise HTTPException(status_code=409, detail="A job was listed more than once.")
|
||||
jobs = []
|
||||
for job_id in job_ids:
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
_check_startable(job)
|
||||
jobs.append(job)
|
||||
views = {jid: _QueueView(j) for jid, j in _jobs.items()}
|
||||
for job in jobs:
|
||||
loop = circular_dependency(views, job["id"])
|
||||
if loop:
|
||||
raise HTTPException(
|
||||
status_code=409,
|
||||
detail="Jobs would wait on each other's last frames in a loop: " + " -> ".join(loop),
|
||||
)
|
||||
for job in jobs:
|
||||
job.pop(_FROM_SNAPSHOT_KEY, None)
|
||||
job.update(status="queued",
|
||||
queued_at=time.time(),
|
||||
error=None,
|
||||
output_path=None,
|
||||
log_file_path=None,
|
||||
started_at=None,
|
||||
finished_at=None,
|
||||
progress=0.0,
|
||||
progress_msg="",
|
||||
phase="queued")
|
||||
# Strictly increasing, so jobs queued together keep their order.
|
||||
others = [j["queued_at"] for j in _jobs.values() if j.get("queued_at") and j is not job]
|
||||
job["queued_at"] = max([job["queued_at"], *[o + 1e-6 for o in others]])
|
||||
_drain_queue()
|
||||
return [_public_job(j) for j in jobs]
|
||||
|
||||
|
||||
@app.post("/api/jobs/queue")
|
||||
def queue_jobs(req: QueueJobsRequest) -> list[dict[str, Any]]:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
return _enqueue(req.job_ids)
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/queue")
|
||||
def queue_job(job_id: str) -> dict[str, Any]:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
return _enqueue([job_id])[0]
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/dequeue")
|
||||
def dequeue_job(job_id: str) -> dict[str, Any]:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
if job["status"] != "queued":
|
||||
raise HTTPException(status_code=409, detail=f"Job is not queued (status={job['status']})")
|
||||
job["status"] = "pending"
|
||||
job["queued_at"] = None
|
||||
_drain_queue()
|
||||
return _public_job(job)
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/start")
|
||||
def start_job(job_id: str) -> dict[str, Any]:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
_advance_job(job)
|
||||
_check_startable(job)
|
||||
_check_dependencies_finished(job)
|
||||
_start(job)
|
||||
return _public_job(job)
|
||||
|
||||
@@ -504,10 +1063,16 @@ def start_job(job_id: str) -> dict[str, Any]:
|
||||
@app.post("/api/jobs/{job_id}/stop")
|
||||
def stop_job(job_id: str) -> dict[str, Any]:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
_advance_job(job)
|
||||
if job["status"] == "queued":
|
||||
job["status"] = "pending"
|
||||
job["queued_at"] = None
|
||||
_drain_queue()
|
||||
return _public_job(job)
|
||||
if job["status"] != "running":
|
||||
raise HTTPException(status_code=409, detail=f"Job is not running (status={job['status']})")
|
||||
job["status"] = "stopped"
|
||||
@@ -519,14 +1084,17 @@ def stop_job(job_id: str) -> dict[str, Any]:
|
||||
@app.delete("/api/jobs/{job_id}")
|
||||
def delete_job(job_id: str) -> dict[str, str]:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
if _jobs.pop(job_id, None) is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
_drain_queue() # clips waiting on this one's last frame can no longer run
|
||||
return {"detail": f"Job {job_id} deleted"}
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/logs")
|
||||
def get_job_logs(job_id: str, after: int = 0) -> dict[str, Any]:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
@@ -545,6 +1113,7 @@ def get_job_logs(job_id: str, after: int = 0) -> dict[str, Any]:
|
||||
@app.get("/api/jobs/{job_id}/video")
|
||||
def get_video(job_id: str) -> FileResponse:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
@@ -552,6 +1121,12 @@ def get_video(job_id: str) -> FileResponse:
|
||||
output_path = job.get("output_path")
|
||||
if job["status"] != "completed" or not output_path:
|
||||
raise HTTPException(status_code=404, detail="No output available for this job")
|
||||
# A job loaded from a --data-dir snapshot may point at a real file this
|
||||
# process never wrote, only reads -- serve it if it's actually there,
|
||||
# otherwise fall back to the same synthetic clip every other job gets.
|
||||
if os.path.isfile(output_path):
|
||||
media_type = "video/mp4" if output_path.endswith(".mp4") else "image/png"
|
||||
return FileResponse(output_path, media_type=media_type, filename=os.path.basename(output_path))
|
||||
# Mirror the real server: image workloads (t2i) output a .png served as an
|
||||
# image; everything else is a video.
|
||||
if output_path.endswith(".png"):
|
||||
@@ -559,9 +1134,69 @@ def get_video(job_id: str) -> FileResponse:
|
||||
return FileResponse(_build_mock_media("mp4"), media_type="video/mp4", filename=f"job_{job_id}.mp4")
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/reference")
|
||||
def get_reference_image(job_id: str, source: str) -> FileResponse:
|
||||
with _state_lock:
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
references = list(job.get("references") or [])
|
||||
return reference_image_response(references, source)
|
||||
|
||||
|
||||
_MERGED_DIR = tempfile.mkdtemp(prefix="fvstudio_merged_")
|
||||
|
||||
|
||||
@app.post("/api/scenes/merge")
|
||||
def merge_scene(req: MergeSceneRequest) -> dict[str, Any]:
|
||||
"""Real merge (same code as the real server) of whatever videos the jobs have.
|
||||
|
||||
Jobs whose video only exists in the mock get the stand-in clip, so the
|
||||
flow can be tried end to end without generating anything.
|
||||
"""
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
picked = []
|
||||
for job_id in req.job_ids:
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
picked.append(dict(job))
|
||||
all_real = all(j.get("output_path") and os.path.isfile(j["output_path"]) for j in picked)
|
||||
stand_in = None if all_real else _build_mock_media("mp4")
|
||||
jobs = [
|
||||
SimpleNamespace(
|
||||
id=j["id"],
|
||||
name=j.get("name", ""),
|
||||
status=j["status"],
|
||||
output_path=(j["output_path"] if all_real else (stand_in if j["status"] == "completed" else None)),
|
||||
) for j in picked
|
||||
]
|
||||
try:
|
||||
result = merge_jobs(jobs, _MERGED_DIR, req.name)
|
||||
except MergeError as e:
|
||||
raise HTTPException(status_code=e.status_code, detail=e.detail) from e
|
||||
return {
|
||||
"filename": result.filename,
|
||||
"url": f"/api/merged/{result.filename}",
|
||||
"clips": result.clips,
|
||||
"seconds": result.seconds,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/merged/{filename}")
|
||||
def get_merged_video(filename: str) -> FileResponse:
|
||||
try:
|
||||
path = merged_path(_MERGED_DIR, filename)
|
||||
except MergeError as e:
|
||||
raise HTTPException(status_code=e.status_code, detail=e.detail) from e
|
||||
return FileResponse(path, media_type="video/mp4", filename=filename)
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/download_log")
|
||||
def download_log(job_id: str) -> PlainTextResponse:
|
||||
with _state_lock:
|
||||
_drain_queue()
|
||||
job = _jobs.get(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
@@ -643,6 +1278,7 @@ def delete_dataset(dataset_id: str) -> dict[str, str]:
|
||||
|
||||
|
||||
def main() -> None:
|
||||
global _state_file
|
||||
import argparse
|
||||
|
||||
import uvicorn
|
||||
@@ -651,8 +1287,29 @@ def main() -> None:
|
||||
parser.add_argument("--host", default="127.0.0.1", help="Bind address (default: 127.0.0.1)")
|
||||
# Default off the real server's 8189 so the mock never shadows a real API.
|
||||
parser.add_argument("--port", type=int, default=8190, help="Port number (default: 8190)")
|
||||
parser.add_argument(
|
||||
"--data-dir",
|
||||
default=None,
|
||||
help=("Read-only: seed jobs/datasets/settings from a real server's "
|
||||
"fastvideo_ui.db under this directory (its --data-dir) instead "
|
||||
"of fake demo data. Nothing the mock does is written back."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--state-file",
|
||||
default=None,
|
||||
help=("JSON file that keeps jobs created or changed in the mock across "
|
||||
"restarts (applied on top of --data-dir). Delete it to reset."),
|
||||
)
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.data_dir is None or not _load_snapshot_from_db(Path(args.data_dir).resolve()):
|
||||
_seed()
|
||||
if args.data_dir is not None:
|
||||
use_server_data_dir(Path(args.data_dir).resolve())
|
||||
if args.state_file:
|
||||
_state_file = Path(args.state_file).resolve()
|
||||
_load_state(_state_file)
|
||||
|
||||
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
|
||||
|
||||
|
||||
|
||||
@@ -6,6 +6,10 @@ from fastvideo_studio.models.create_job_request import CreateJobRequest
|
||||
from fastvideo_studio.models.settings_update import SettingsUpdate
|
||||
from fastvideo_studio.models.create_dataset_request import CreateDatasetRequest
|
||||
from fastvideo_studio.models.update_caption_request import UpdateCaptionRequest
|
||||
from fastvideo_studio.models.queue_jobs_request import QueueJobsRequest
|
||||
from fastvideo_studio.models.merge_scene_request import MergeSceneRequest
|
||||
from fastvideo_studio.models.trim_request import GradeSegment, TrimRequest
|
||||
from fastvideo_studio.models.scene_draft_request import ClipRevisionRequest, SceneDraftReference, SceneDraftRequest
|
||||
|
||||
|
||||
def model_label(model_path: str) -> str:
|
||||
@@ -18,5 +22,12 @@ __all__ = [
|
||||
"SettingsUpdate",
|
||||
"CreateDatasetRequest",
|
||||
"UpdateCaptionRequest",
|
||||
"QueueJobsRequest",
|
||||
"MergeSceneRequest",
|
||||
"GradeSegment",
|
||||
"TrimRequest",
|
||||
"ClipRevisionRequest",
|
||||
"SceneDraftReference",
|
||||
"SceneDraftRequest",
|
||||
"model_label",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,11 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Request model for merging a scene's clips into one video."""
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class MergeSceneRequest(BaseModel):
|
||||
#: Completed jobs, in the order their videos should play.
|
||||
job_ids: list[str]
|
||||
#: Used to name the merged file.
|
||||
name: str = ""
|
||||
@@ -0,0 +1,8 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Request model for queueing jobs."""
|
||||
|
||||
from pydantic import BaseModel
|
||||
|
||||
|
||||
class QueueJobsRequest(BaseModel):
|
||||
job_ids: list[str]
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Request model for drafting a scene's jobs with the scene writer."""
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
|
||||
class SceneDraftReference(BaseModel):
|
||||
path: str
|
||||
media_type: Literal["image", "audio", "video"]
|
||||
description: str = ""
|
||||
|
||||
|
||||
class SceneDraftRequest(BaseModel):
|
||||
script: str = Field(min_length=1)
|
||||
scene_name: str = Field(pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
references: list[SceneDraftReference] = Field(default_factory=list)
|
||||
instructions: str = ""
|
||||
# Let the writer download reference photos from Wikimedia Commons for subjects with none.
|
||||
find_references: bool = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _needs_references(self) -> "SceneDraftRequest":
|
||||
if not self.references and not self.find_references:
|
||||
raise ValueError("Give at least one reference file, or let the writer find references online.")
|
||||
return self
|
||||
|
||||
|
||||
class ClipRevisionRequest(BaseModel):
|
||||
"""Revise one clip from feedback on its generated take; `job` and `neighbors` are Studio jobs."""
|
||||
|
||||
job: dict[str, Any]
|
||||
feedback: str = Field(min_length=1)
|
||||
# The clips before and after it in the scene, for continuity.
|
||||
neighbors: list[dict[str, Any]] = Field(default_factory=list)
|
||||
@@ -0,0 +1,30 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Request model for cutting a job's video and color-grading it section by section."""
|
||||
|
||||
from pydantic import BaseModel, Field
|
||||
|
||||
|
||||
class GradeSegment(BaseModel):
|
||||
"""One razor-cut section of the kept range, and its own color grade.
|
||||
|
||||
``end_seconds`` is seconds into the *kept* (already start/end-trimmed)
|
||||
range -- the same timeline the preview video plays -- not the original
|
||||
file's. Only the last section in a list may leave it ``None``, meaning
|
||||
"to the end"; sections must be given in increasing order and cover the
|
||||
whole kept range, with no gaps or reordering.
|
||||
"""
|
||||
end_seconds: float | None = None
|
||||
#: Added directly to 0-255 pixel values. 0 = unchanged.
|
||||
brightness: float = 0.0
|
||||
#: 1.0 = unchanged, 0 = flat gray, >1 = more contrast.
|
||||
contrast: float = 1.0
|
||||
#: 1.0 = unchanged, 0 = grayscale, >1 = more saturated.
|
||||
saturation: float = 1.0
|
||||
|
||||
|
||||
class TrimRequest(BaseModel):
|
||||
start_seconds: float = 0.0
|
||||
#: None keeps to the end of the video.
|
||||
end_seconds: float | None = None
|
||||
#: Defaults to one neutral section covering the whole kept range, i.e. a plain trim with no grading.
|
||||
segments: list[GradeSegment] = Field(default_factory=lambda: [GradeSegment()])
|
||||
@@ -4,10 +4,21 @@ import type { NextConfig } from 'next';
|
||||
|
||||
const configDir = path.dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
// Where /api/* is proxied when the frontend is built with
|
||||
// NEXT_PUBLIC_API_BASE_URL=/api, so the web app and API share one origin --
|
||||
// e.g. behind a single ngrok tunnel. Read at build time.
|
||||
const apiProxyTarget = (process.env.STUDIO_API_PROXY_TARGET || 'http://127.0.0.1:8189').replace(/\/+$/, '');
|
||||
|
||||
const nextConfig: NextConfig = {
|
||||
// Point tracing at the monorepo root (apps/fastvideo_studio -> apps -> repo
|
||||
// root) so Next stops warning about multiple lockfiles in the workspace.
|
||||
outputFileTracingRoot: path.join(configDir, '..', '..'),
|
||||
// `next dev` blocks its dev-only assets/HMR for pages opened from a host
|
||||
// other than localhost; allow the ngrok tunnel's domains. No effect in prod.
|
||||
allowedDevOrigins: ['*.ngrok-free.app', '*.ngrok.app', '*.ngrok.io'],
|
||||
async rewrites() {
|
||||
return [{ source: '/api/:path*', destination: `${apiProxyTarget}/api/:path*` }];
|
||||
},
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Serve a job's image reference so the job form can show a thumbnail of it.
|
||||
|
||||
Only a file the job itself lists as an image reference is served: the API may
|
||||
be reachable from outside (a tunnel), so a path from the request alone never
|
||||
opens a file.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import mimetypes
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".webp", ".bmp"}
|
||||
|
||||
|
||||
def reference_image_response(references: list[dict[str, Any]] | None, source: str) -> FileResponse:
|
||||
listed = any(r.get("source") == source and r.get("media_type") == "image" for r in references or [])
|
||||
ext = os.path.splitext(source)[1].lower()
|
||||
if not listed or ext not in IMAGE_EXTENSIONS:
|
||||
raise HTTPException(status_code=404, detail="No such image reference on this job")
|
||||
if not os.path.isfile(source):
|
||||
raise HTTPException(status_code=404, detail="Reference file not found on disk")
|
||||
return FileResponse(source, media_type=mimetypes.guess_type(source)[0] or "application/octet-stream")
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Draft MiniMax-H3 jobs from a script with a DeepSeek Harness agent."""
|
||||
@@ -0,0 +1,2 @@
|
||||
dist/
|
||||
node_modules/
|
||||
@@ -0,0 +1,5 @@
|
||||
# Mounts the h3_write_job tool. Each scene-writer run replaces this row's
|
||||
# config (outDir, references) with an id-targeted patch.
|
||||
- insert:
|
||||
- id: h3-scene-writer
|
||||
name: fastvideo-h3-scene-writer
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* DSH plugin for FastVideo Studio's scene writer. `h3_write_job` validates a
|
||||
* drafted MiniMax-H3 job against the prompt format and writes it into the
|
||||
* run's output directory; the agent learns the format from the
|
||||
* `h3-job-writer` skill, and this tool is the hard check. When the run has a
|
||||
* reference directory, `find_reference_image` also lets the agent download
|
||||
* photos from Wikimedia Commons into it for subjects nobody gave a reference
|
||||
* for; those files then count as allowed references.
|
||||
*/
|
||||
|
||||
import { mkdir, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
import Schema from '@deepseek-ai/schemastery'
|
||||
|
||||
import { fetchReferenceImages } from './reference-images.ts'
|
||||
import { validateJob } from './validate-job.ts'
|
||||
|
||||
export const name = 'h3-scene-writer'
|
||||
export const inject = ['tools']
|
||||
|
||||
export interface Config {
|
||||
/** Directory the validated job JSONs are written to; set per run. */
|
||||
outDir: string
|
||||
/** Absolute paths of the reference files a job may use. */
|
||||
references: string[]
|
||||
/** The video references among them that carry a soundtrack (each takes an <Audio N> label too). */
|
||||
soundVideos: string[]
|
||||
/** Where `find_reference_image` saves downloads; empty leaves the tool out. */
|
||||
refsDir: string
|
||||
/** When set (revising one clip), every written job must keep this name. */
|
||||
expectedName: string
|
||||
}
|
||||
|
||||
export const Config: Schema<Config> = Schema.object({
|
||||
outDir: Schema.string().default(''),
|
||||
references: Schema.array(Schema.string()).default([]),
|
||||
soundVideos: Schema.array(Schema.string()).default([]),
|
||||
refsDir: Schema.string().default(''),
|
||||
expectedName: Schema.string().default(''),
|
||||
})
|
||||
|
||||
export function apply(ctx: Context, config: Config) {
|
||||
const allowed = new Set(config.references)
|
||||
|
||||
ctx.tools.register(defineTool({
|
||||
name: 'h3_write_job',
|
||||
description:
|
||||
'Validate one MiniMax-H3 job (the full job JSON as a string) and, if it passes, write it as <file_name>.json in the output directory. '
|
||||
+ 'Returns the list of problems when it does not pass; fix them and call again.',
|
||||
parameters: {
|
||||
file_name: { type: 'string', required: true, description: 'File stem, e.g. "whiplash_rehearsal_clip_01"' },
|
||||
job_json: { type: 'string', required: true, description: 'The complete job object as JSON text' },
|
||||
},
|
||||
output: {
|
||||
schema: { type: 'string' },
|
||||
render: (_args, value) => [{ type: 'text', text: value }],
|
||||
},
|
||||
async execute(args) {
|
||||
if (!config.outDir) {
|
||||
return 'Error: no output directory is configured for this run; nothing can be written.'
|
||||
}
|
||||
if (!/^[A-Za-z0-9_-]{1,100}$/.test(args.file_name)) {
|
||||
return 'Error: file_name must be letters, digits, "_" or "-" only.'
|
||||
}
|
||||
let job: unknown
|
||||
try {
|
||||
job = JSON.parse(args.job_json)
|
||||
} catch (error) {
|
||||
return `Error: job_json is not valid JSON: ${(error as Error).message}`
|
||||
}
|
||||
const problems = validateJob(job, allowed, new Set(config.soundVideos))
|
||||
if (config.expectedName && (job as { name?: unknown }).name !== config.expectedName) {
|
||||
problems.push(`"name" must stay "${config.expectedName}": a revision is a new take of the same clip.`)
|
||||
}
|
||||
if (problems.length > 0) {
|
||||
return `Not written. Fix these problems and call h3_write_job again:\n- ${problems.join('\n- ')}`
|
||||
}
|
||||
await mkdir(config.outDir, { recursive: true })
|
||||
const file = path.join(config.outDir, `${args.file_name}.json`)
|
||||
await writeFile(file, `${JSON.stringify(job, null, 2)}\n`)
|
||||
return `Wrote ${file}.`
|
||||
},
|
||||
}))
|
||||
|
||||
if (!config.refsDir) return
|
||||
ctx.tools.register(defineTool({
|
||||
name: 'find_reference_image',
|
||||
description:
|
||||
'Search Wikimedia Commons for photos of a person, place, or object that has no reference file, and download up to '
|
||||
+ '`count` candidates as image references. Returns each file\'s path with its title, size, and license. '
|
||||
+ 'Look at each candidate with read_image before using it (Commons results include paintings and group photos). '
|
||||
+ 'Use the chosen paths in "references" with media_type "image".',
|
||||
parameters: {
|
||||
subject: { type: 'string', required: true, description: 'What the reference is for, e.g. "Robert Pattinson" or "jazz rehearsal room"' },
|
||||
query: { type: 'string', required: true, description: 'Search words, e.g. the person\'s full name, or "rehearsal room interior"' },
|
||||
count: { type: 'number', description: 'Candidates to download, 1 to 4 (default 3)' },
|
||||
},
|
||||
output: {
|
||||
schema: { type: 'string' },
|
||||
render: (_args, value) => [{ type: 'text', text: value }],
|
||||
},
|
||||
async execute(args) {
|
||||
const count = Math.min(4, Math.max(1, Math.round(args.count ?? 3)))
|
||||
let images
|
||||
try {
|
||||
await mkdir(config.refsDir, { recursive: true })
|
||||
images = await fetchReferenceImages(config.refsDir, args.subject, args.query, count)
|
||||
} catch (error) {
|
||||
return `Error: ${(error as Error).message}`
|
||||
}
|
||||
if (images.length === 0) {
|
||||
return `No usable photos found on Wikimedia Commons for "${args.query}". Try other words (a full name, a simpler description), or write the job without a reference for this subject.`
|
||||
}
|
||||
for (const image of images) allowed.add(image.path)
|
||||
return [
|
||||
`Downloaded ${images.length} candidate(s) for ${args.subject}:`,
|
||||
...images.map((i) => `- ${i.path} - "${i.title}", ${i.width}x${i.height}, ${i.license}`),
|
||||
'Use one per subject: prefer a clear, recent, front-facing photo of just that subject (titles mentioning "cropped" or "portrait" are often best).',
|
||||
].join('\n')
|
||||
},
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "fastvideo-h3-scene-writer",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "DSH bundle for FastVideo Studio's scene writer: the h3_write_job and find_reference_image tools.",
|
||||
"type": "module",
|
||||
"main": "./dist/h3-scene-writer.js",
|
||||
"exports": {
|
||||
".": "./dist/h3-scene-writer.js"
|
||||
},
|
||||
"files": [
|
||||
"dist/h3-scene-writer.js",
|
||||
"cordis.patch.yml"
|
||||
],
|
||||
"dsh": {
|
||||
"bundle": {
|
||||
"patch": "./cordis.patch.yml"
|
||||
}
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/cordis": "*",
|
||||
"@deepseek-ai/dsh-tools": "*",
|
||||
"@deepseek-ai/schemastery": "*"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "esbuild h3-scene-writer.ts --bundle --external:@deepseek-ai/* --format=esm --platform=node --target=node22 --outfile=dist/h3-scene-writer.js"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import { existsSync, mkdtempSync, readFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { fetchReferenceImages, slug } from './reference-images.ts'
|
||||
|
||||
function page(index: number, title: string, mime: string, width: number, height: number) {
|
||||
return {
|
||||
index,
|
||||
title: `File:${title}`,
|
||||
imageinfo: [{
|
||||
mime,
|
||||
width,
|
||||
height,
|
||||
thumburl: `https://upload.example/thumb/${encodeURIComponent(title)}`.replace(/\.png$/, '.png'),
|
||||
thumbwidth: 1024,
|
||||
thumbheight: Math.round((1024 * height) / width),
|
||||
descriptionurl: `https://commons.wikimedia.org/wiki/File:${encodeURIComponent(title)}`,
|
||||
extmetadata: {
|
||||
LicenseShortName: { value: 'CC BY-SA 4.0' },
|
||||
Artist: { value: '<a href="//commons.wikimedia.org/wiki/User:Someone">Someone</a>' },
|
||||
},
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
function stubCommons(pages: unknown[]) {
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.startsWith('https://commons.wikimedia.org/w/api.php')) {
|
||||
return new Response(JSON.stringify({ query: { pages: Object.fromEntries(pages.map((p, i) => [String(i), p])) } }))
|
||||
}
|
||||
return new Response(new Uint8Array([1, 2, 3]))
|
||||
})
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
return fetchMock
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
describe('fetchReferenceImages', () => {
|
||||
it('downloads the best photo matches and records their sources', async () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), 'h3-refs-'))
|
||||
stubCommons([
|
||||
page(2, 'Second.jpg', 'image/jpeg', 2000, 3000),
|
||||
page(1, 'First portrait (cropped).jpg', 'image/jpeg', 1200, 1600),
|
||||
page(3, 'Tiny.jpg', 'image/jpeg', 300, 200),
|
||||
page(4, 'Drawing.svg', 'image/svg+xml', 2000, 2000),
|
||||
])
|
||||
|
||||
const images = await fetchReferenceImages(dir, 'Robert Pattinson', 'Robert Pattinson', 3)
|
||||
|
||||
// Search order kept; too-small and non-photo files skipped.
|
||||
expect(images.map((i) => i.title)).toEqual(['First portrait (cropped).jpg', 'Second.jpg'])
|
||||
expect(images.map((i) => path.basename(i.path))).toEqual(['robert_pattinson_1.jpg', 'robert_pattinson_2.jpg'])
|
||||
expect(images.every((i) => existsSync(i.path))).toBe(true)
|
||||
expect(images[0]).toMatchObject({ license: 'CC BY-SA 4.0', artist: 'Someone', width: 1024, height: 1365 })
|
||||
const sources = JSON.parse(readFileSync(path.join(dir, 'sources.json'), 'utf8'))
|
||||
expect(sources).toEqual(images)
|
||||
})
|
||||
|
||||
it('numbers a second search for the same subject after the first', async () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), 'h3-refs-'))
|
||||
stubCommons([page(1, 'A.jpg', 'image/jpeg', 800, 800)])
|
||||
await fetchReferenceImages(dir, 'Robert Pattinson', 'Robert Pattinson', 1)
|
||||
const again = await fetchReferenceImages(dir, 'Robert Pattinson', 'Robert Pattinson actor', 1)
|
||||
|
||||
expect(path.basename(again[0].path)).toBe('robert_pattinson_2.jpg')
|
||||
expect(JSON.parse(readFileSync(path.join(dir, 'sources.json'), 'utf8'))).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('reports a failed search', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response('', { status: 503 })))
|
||||
await expect(fetchReferenceImages(mkdtempSync(path.join(tmpdir(), 'h3-refs-')), 'x', 'x', 1))
|
||||
.rejects.toThrow(/HTTP 503/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('slug', () => {
|
||||
it('makes a safe file stem', () => {
|
||||
expect(slug('Fletcher\'s rehearsal room!')).toBe('fletcher_s_rehearsal_room')
|
||||
expect(slug('***')).toBe('reference')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* Reference image search for the scene writer: finds photos on Wikimedia
|
||||
* Commons, downloads them into the run's reference directory, and records
|
||||
* where each came from (page, license, author) in `sources.json` beside them
|
||||
* so the person reviewing the draft can check and credit them.
|
||||
*/
|
||||
|
||||
import { readFile, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
const COMMONS_API = 'https://commons.wikimedia.org/w/api.php'
|
||||
// Wikimedia asks API clients to identify themselves.
|
||||
const USER_AGENT = 'FastVideoStudio-SceneWriter/0.1 (reference image search)'
|
||||
const DOWNLOAD_WIDTH = 1024
|
||||
const MIN_SIDE = 400
|
||||
const TIMEOUT_MS = 20_000
|
||||
|
||||
export interface FetchedImage {
|
||||
path: string
|
||||
subject: string
|
||||
query: string
|
||||
title: string
|
||||
page: string
|
||||
license: string
|
||||
artist: string
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
interface CommonsPage {
|
||||
index: number
|
||||
title: string
|
||||
imageinfo?: {
|
||||
mime: string
|
||||
width: number
|
||||
height: number
|
||||
thumburl?: string
|
||||
thumbwidth?: number
|
||||
thumbheight?: number
|
||||
descriptionurl: string
|
||||
extmetadata?: Record<string, { value?: string } | undefined>
|
||||
}[]
|
||||
}
|
||||
|
||||
function plainText(html: string | undefined): string {
|
||||
return (html ?? '').replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim()
|
||||
}
|
||||
|
||||
export function slug(text: string): string {
|
||||
return text.toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '').slice(0, 40) || 'reference'
|
||||
}
|
||||
|
||||
/** Commons photos (JPEG/PNG, at least MIN_SIDE px on the short side) for a search, best match first. */
|
||||
export async function searchCommons(query: string, limit: number): Promise<CommonsPage[]> {
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'search',
|
||||
gsrsearch: `${query} filetype:bitmap`,
|
||||
gsrnamespace: '6',
|
||||
gsrlimit: '20',
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|size|mime|extmetadata',
|
||||
iiurlwidth: String(DOWNLOAD_WIDTH),
|
||||
})
|
||||
const response = await fetch(`${COMMONS_API}?${params}`, {
|
||||
headers: { 'user-agent': USER_AGENT },
|
||||
signal: AbortSignal.timeout(TIMEOUT_MS),
|
||||
})
|
||||
if (!response.ok) throw new Error(`Wikimedia Commons search failed: HTTP ${response.status}`)
|
||||
const body = await response.json() as { query?: { pages?: Record<string, CommonsPage> } }
|
||||
return Object.values(body.query?.pages ?? {})
|
||||
.filter((p) => {
|
||||
const info = p.imageinfo?.[0]
|
||||
return info !== undefined && ['image/jpeg', 'image/png'].includes(info.mime)
|
||||
&& Math.min(info.width, info.height) >= MIN_SIDE && info.thumburl !== undefined
|
||||
})
|
||||
.sort((a, b) => a.index - b.index)
|
||||
.slice(0, limit)
|
||||
}
|
||||
|
||||
/**
|
||||
* Search, download up to `count` candidates into `refsDir`, and append them to
|
||||
* `refsDir/sources.json`. Returns the downloaded images.
|
||||
*/
|
||||
export async function fetchReferenceImages(
|
||||
refsDir: string,
|
||||
subject: string,
|
||||
query: string,
|
||||
count: number,
|
||||
): Promise<FetchedImage[]> {
|
||||
const pages = await searchCommons(query, count)
|
||||
const fetched: FetchedImage[] = []
|
||||
const sourcesFile = path.join(refsDir, 'sources.json')
|
||||
let sources: FetchedImage[] = []
|
||||
try {
|
||||
sources = JSON.parse(await readFile(sourcesFile, 'utf8')) as FetchedImage[]
|
||||
} catch (error) {
|
||||
// ENOENT: the first image of this run; anything else would surface on the write below.
|
||||
void error
|
||||
}
|
||||
for (const page of pages) {
|
||||
const info = page.imageinfo![0]
|
||||
const response = await fetch(info.thumburl!, {
|
||||
headers: { 'user-agent': USER_AGENT },
|
||||
signal: AbortSignal.timeout(TIMEOUT_MS),
|
||||
})
|
||||
if (!response.ok) continue
|
||||
const ext = path.extname(new URL(info.thumburl!).pathname).toLowerCase() === '.png' ? '.png' : '.jpg'
|
||||
let file = ''
|
||||
for (let n = 1; ; n++) {
|
||||
file = path.join(refsDir, `${slug(subject)}_${n}${ext}`)
|
||||
if (!sources.some((s) => s.path === file) && !fetched.some((s) => s.path === file)) break
|
||||
}
|
||||
await writeFile(file, Buffer.from(await response.arrayBuffer()))
|
||||
const meta = info.extmetadata ?? {}
|
||||
fetched.push({
|
||||
path: file,
|
||||
subject,
|
||||
query,
|
||||
title: page.title.replace(/^File:/, ''),
|
||||
page: info.descriptionurl,
|
||||
license: plainText(meta.LicenseShortName?.value) || 'unknown license',
|
||||
artist: plainText(meta.Artist?.value) || 'unknown author',
|
||||
width: info.thumbwidth ?? info.width,
|
||||
height: info.thumbheight ?? info.height,
|
||||
})
|
||||
}
|
||||
await writeFile(sourcesFile, `${JSON.stringify([...sources, ...fetched], null, 2)}\n`)
|
||||
return fetched
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { validateJob } from './validate-job.ts'
|
||||
|
||||
const dir = mkdtempSync(path.join(tmpdir(), 'h3-validate-'))
|
||||
const picture = path.join(dir, 'a.png')
|
||||
const voice = path.join(dir, 'a.mp3')
|
||||
const clip = path.join(dir, 'tail.mp4')
|
||||
writeFileSync(picture, '')
|
||||
writeFileSync(voice, '')
|
||||
writeFileSync(clip, '')
|
||||
|
||||
function goodJob() {
|
||||
return {
|
||||
model_id: 'MiniMaxAI/MiniMax-H3',
|
||||
name: 'scene-clip-01',
|
||||
workload_type: 'i2v',
|
||||
prompt: [
|
||||
'subject_definitions:',
|
||||
'<Subject 1> is Ada from <Picture 1>.',
|
||||
'<Audio 1> is the voice-timbre reference for <Subject 1> (S1), containing a spoken English vocal layer.',
|
||||
'',
|
||||
'summary:',
|
||||
'[reference generation + audio reference] Ada talks.',
|
||||
'',
|
||||
'retention_analysis:',
|
||||
'<Subject 1> (appears in [Shot 1], [Shot 2]): fully_preserved - her face.',
|
||||
'',
|
||||
'detailed_description:',
|
||||
'[Shot 1] Medium shot. (S1) <d>[English] Hello.</d>',
|
||||
'[Shot 2] At 00:03.000, the shot cuts to a close-up shot. (S1) <d>[English] Bye.</d> <cutoff>',
|
||||
'',
|
||||
'overall_soundscape:',
|
||||
'Room tone.',
|
||||
'',
|
||||
'non_diegetic_music:',
|
||||
'N/A',
|
||||
],
|
||||
references: [
|
||||
{ source: picture, media_type: 'image' },
|
||||
{ source: voice, media_type: 'audio' },
|
||||
],
|
||||
negative_prompt: '',
|
||||
guidance_scale: 1,
|
||||
num_frames: 144,
|
||||
fps: 24,
|
||||
height: 768,
|
||||
width: 1344,
|
||||
num_inference_steps: 50,
|
||||
seed: 0,
|
||||
num_gpus: 4,
|
||||
use_fsdp_inference: true,
|
||||
}
|
||||
}
|
||||
|
||||
const allowed = new Set([picture, voice, clip])
|
||||
|
||||
describe('validateJob', () => {
|
||||
it('passes a well-formed job', () => {
|
||||
expect(validateJob(goodJob(), allowed)).toEqual([])
|
||||
})
|
||||
|
||||
it('rejects clips longer than one generation', () => {
|
||||
expect(validateJob({ ...goodJob(), num_frames: 400 }, allowed).join()).toMatch(/num_frames/)
|
||||
})
|
||||
|
||||
it('rejects a tag that points past the reference list', () => {
|
||||
const job = goodJob()
|
||||
job.prompt[1] = '<Subject 1> is Ada from <Picture 2>.'
|
||||
expect(validateJob(job, allowed).join()).toMatch(/<Picture 2> but there are only 1/)
|
||||
})
|
||||
|
||||
it('rejects references that were not provided', () => {
|
||||
const job = goodJob()
|
||||
job.references[0] = { source: '/elsewhere/b.png', media_type: 'image' }
|
||||
expect(validateJob(job, allowed).join()).toMatch(/not one of the reference files/)
|
||||
})
|
||||
|
||||
it('rejects a missing section and cuts past the end of the clip', () => {
|
||||
const job = goodJob()
|
||||
job.prompt = job.prompt.filter((l) => l !== 'overall_soundscape:')
|
||||
job.num_frames = 48
|
||||
const problems = validateJob(job, allowed).join('\n')
|
||||
expect(problems).toMatch(/missing the "overall_soundscape:"/)
|
||||
expect(problems).toMatch(/cuts at 00:03.000, but the clip is only 2.00 s/)
|
||||
})
|
||||
|
||||
it('rejects appearance lists naming shots that do not exist', () => {
|
||||
const job = goodJob()
|
||||
job.prompt[8] = '<Subject 1> (appears in [Shot 1], [Shot 3]): fully_preserved - her face.'
|
||||
expect(validateJob(job, allowed).join()).toMatch(/\[Shot 3\], but there are only 2/)
|
||||
})
|
||||
|
||||
it('requires the audio summary tag to match the references', () => {
|
||||
const job = goodJob()
|
||||
job.prompt[5] = '[reference generation] Ada talks.'
|
||||
expect(validateJob(job, allowed).join()).toMatch(/must include "\+ audio reference\]"/)
|
||||
})
|
||||
|
||||
it('numbers a sound video\'s soundtrack before later audio files', () => {
|
||||
const job = goodJob()
|
||||
job.references = [{ source: clip, media_type: 'video' }, ...job.references]
|
||||
job.prompt[1] = '<Subject 1> is Ada from <Picture 1>, continuing from <Video 1>.'
|
||||
// The video's soundtrack is <Audio 1>, so the voice file is <Audio 2>.
|
||||
expect(validateJob(job, allowed, new Set([clip])).join()).toMatch(/<Audio 2> is in "references" but never described/)
|
||||
job.prompt = job.prompt.map((l) => l.replaceAll('<Audio 1>', '<Audio 2>'))
|
||||
expect(validateJob(job, allowed, new Set([clip]))).toEqual([])
|
||||
// Without a soundtrack the voice file stays <Audio 1>.
|
||||
expect(validateJob(job, allowed).join()).toMatch(/<Audio 2> but there are only 1/)
|
||||
})
|
||||
|
||||
it('enforces the pipeline\'s reference limits', () => {
|
||||
const job = goodJob()
|
||||
job.references = [{ source: voice, media_type: 'audio' }]
|
||||
expect(validateJob(job, allowed).join()).toMatch(/paired with at least one image or video/)
|
||||
job.references = Array.from({ length: 10 }, () => ({ source: picture, media_type: 'image' }))
|
||||
expect(validateJob(job, allowed).join()).toMatch(/at most 9 image references/)
|
||||
})
|
||||
|
||||
it('allows only listed references, and none when the list is empty', () => {
|
||||
expect(validateJob(goodJob(), new Set()).join()).toMatch(/not one of the reference files/)
|
||||
expect(validateJob(goodJob(), null)).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* Structural checks for a drafted MiniMax-H3 job. Each check is something a
|
||||
* model draft has actually gotten wrong, or something that silently breaks a
|
||||
* generation: a missing reference, a tag pointing past the reference list,
|
||||
* appearance lists naming shots that don't exist, cuts after the clip ends.
|
||||
*/
|
||||
|
||||
import { existsSync } from 'node:fs'
|
||||
|
||||
const SECTIONS = [
|
||||
'subject_definitions:',
|
||||
'summary:',
|
||||
'retention_analysis:',
|
||||
'detailed_description:',
|
||||
'overall_soundscape:',
|
||||
'non_diegetic_music:',
|
||||
] as const
|
||||
|
||||
export const MAX_FRAMES = 345
|
||||
export const FPS = 24
|
||||
|
||||
// Ref2VA limits enforced by FastVideo's MiniMax-H3 pipeline (reference.py).
|
||||
const MAX_REFERENCES = { image: 9, video: 3, audio: 3 } as const
|
||||
const MAX_TOTAL_REFERENCES = 12
|
||||
|
||||
const REQUIRED_FIELDS: Record<string, string> = {
|
||||
model_id: 'string',
|
||||
name: 'string',
|
||||
workload_type: 'string',
|
||||
num_frames: 'number',
|
||||
fps: 'number',
|
||||
height: 'number',
|
||||
width: 'number',
|
||||
num_inference_steps: 'number',
|
||||
seed: 'number',
|
||||
num_gpus: 'number',
|
||||
}
|
||||
|
||||
interface Reference {
|
||||
source: string
|
||||
media_type: string
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function highestTag(text: string, kind: string): number {
|
||||
let max = 0
|
||||
for (const m of text.matchAll(new RegExp(`<${kind} (\\d+)>`, 'g'))) max = Math.max(max, Number(m[1]))
|
||||
return max
|
||||
}
|
||||
|
||||
function usedTags(text: string, kind: string): Set<number> {
|
||||
return new Set([...text.matchAll(new RegExp(`<${kind} (\\d+)>`, 'g'))].map((m) => Number(m[1])))
|
||||
}
|
||||
|
||||
function seconds(stamp: string): number {
|
||||
const [mm, ss] = stamp.split(':')
|
||||
return Number(mm) * 60 + Number(ss)
|
||||
}
|
||||
|
||||
/**
|
||||
* Every problem found, as model-facing sentences; empty when the job passes.
|
||||
* `allowedReferences` lists the files a job may use (`null` allows any file that exists).
|
||||
* `soundVideos` are the video references that carry a soundtrack: H3 gives
|
||||
* each of them the next `<Audio N>` label, ahead of its `<Video N>`.
|
||||
*/
|
||||
export function validateJob(
|
||||
job: unknown,
|
||||
allowedReferences: ReadonlySet<string> | null,
|
||||
soundVideos: ReadonlySet<string> = new Set(),
|
||||
): string[] {
|
||||
const problems: string[] = []
|
||||
if (!isRecord(job)) return ['The job must be a JSON object.']
|
||||
|
||||
for (const [field, type] of Object.entries(REQUIRED_FIELDS)) {
|
||||
if (typeof job[field] !== type) problems.push(`"${field}" must be a ${type}.`)
|
||||
}
|
||||
if (job.model_id !== 'MiniMaxAI/MiniMax-H3') problems.push('"model_id" must be "MiniMaxAI/MiniMax-H3".')
|
||||
if (typeof job.name === 'string' && !/-clip-\d{2}$/.test(job.name)) {
|
||||
problems.push('"name" must end in "-clip-NN" (e.g. "whiplash-rehearsal-clip-01") so Studio groups the clips into a scene.')
|
||||
}
|
||||
if (job.fps !== FPS) problems.push(`"fps" must be ${FPS}.`)
|
||||
const frames = typeof job.num_frames === 'number' ? job.num_frames : 0
|
||||
if (!Number.isInteger(frames) || frames < 1 || frames > MAX_FRAMES) {
|
||||
problems.push(`"num_frames" must be a whole number from 1 to ${MAX_FRAMES} (${MAX_FRAMES / FPS} s at ${FPS} fps).`)
|
||||
}
|
||||
const duration = frames / FPS
|
||||
|
||||
// References
|
||||
const refs: Reference[] = []
|
||||
if (!Array.isArray(job.references) || job.references.length === 0) {
|
||||
problems.push('"references" must be a non-empty list of {"source", "media_type"}.')
|
||||
} else {
|
||||
for (const [i, ref] of job.references.entries()) {
|
||||
if (!isRecord(ref) || typeof ref.source !== 'string' || typeof ref.media_type !== 'string') {
|
||||
problems.push(`references[${i}] must be {"source": string, "media_type": string}.`)
|
||||
continue
|
||||
}
|
||||
if (!['image', 'audio', 'video'].includes(ref.media_type)) {
|
||||
problems.push(`references[${i}].media_type must be "image", "audio" or "video".`)
|
||||
}
|
||||
if (allowedReferences !== null && !allowedReferences.has(ref.source)) {
|
||||
problems.push(`references[${i}].source "${ref.source}" is not one of the reference files you were given.`)
|
||||
} else if (!existsSync(ref.source)) {
|
||||
problems.push(`references[${i}].source "${ref.source}" does not exist.`)
|
||||
}
|
||||
refs.push({ source: ref.source, media_type: ref.media_type })
|
||||
}
|
||||
for (const [type, limit] of Object.entries(MAX_REFERENCES)) {
|
||||
const n = refs.filter((r) => r.media_type === type).length
|
||||
if (n > limit) problems.push(`H3 accepts at most ${limit} ${type} references; this job has ${n}.`)
|
||||
}
|
||||
if (refs.length > MAX_TOTAL_REFERENCES) {
|
||||
problems.push(`H3 accepts at most ${MAX_TOTAL_REFERENCES} references in total; this job has ${refs.length}.`)
|
||||
}
|
||||
if (refs.length > 0 && refs.every((r) => r.media_type === 'audio')) {
|
||||
problems.push('Audio references must be paired with at least one image or video reference.')
|
||||
}
|
||||
}
|
||||
|
||||
// Prompt
|
||||
if (!Array.isArray(job.prompt) || !job.prompt.every((line) => typeof line === 'string')) {
|
||||
problems.push('"prompt" must be a list of strings (one line each).')
|
||||
return problems
|
||||
}
|
||||
const lines = job.prompt as string[]
|
||||
const text = lines.join('\n')
|
||||
|
||||
const positions = SECTIONS.map((s) => lines.indexOf(s))
|
||||
SECTIONS.forEach((s, i) => {
|
||||
if (positions[i] === -1) problems.push(`The prompt is missing the "${s}" section header line.`)
|
||||
})
|
||||
if (positions.every((p) => p !== -1) && positions.some((p, i) => i > 0 && p < positions[i - 1])) {
|
||||
problems.push(`The prompt sections must appear in this order: ${SECTIONS.join(', ')}.`)
|
||||
}
|
||||
|
||||
// Tags must match the reference list, numbered the way the pipeline labels
|
||||
// it: one counter per kind in list order, and a video with a soundtrack
|
||||
// takes the next <Audio N> as well as its <Video N>.
|
||||
const counts = { Picture: 0, Audio: 0, Video: 0 }
|
||||
const soundtrackLabels: string[] = []
|
||||
let audioFiles = 0
|
||||
for (const ref of refs) {
|
||||
if (ref.media_type === 'image') counts.Picture++
|
||||
else if (ref.media_type === 'audio') {
|
||||
counts.Audio++
|
||||
audioFiles++
|
||||
} else if (ref.media_type === 'video') {
|
||||
counts.Video++
|
||||
if (soundVideos.has(ref.source)) {
|
||||
counts.Audio++
|
||||
soundtrackLabels.push(`<Audio ${counts.Audio}> (the soundtrack of <Video ${counts.Video}>)`)
|
||||
}
|
||||
}
|
||||
}
|
||||
const soundtrackNumbers = new Set(soundtrackLabels.map((l) => Number(l.match(/^<Audio (\d+)>/)![1])))
|
||||
for (const kind of ['Picture', 'Audio', 'Video'] as const) {
|
||||
const highest = highestTag(text, kind)
|
||||
if (highest > counts[kind]) {
|
||||
problems.push(`The prompt mentions <${kind} ${highest}> but there are only ${counts[kind]} ${kind.toLowerCase()} label(s).`)
|
||||
}
|
||||
const used = usedTags(text, kind)
|
||||
for (let k = 1; k <= counts[kind]; k++) {
|
||||
if (kind === 'Audio' && soundtrackNumbers.has(k)) continue
|
||||
if (!used.has(k)) problems.push(`<${kind} ${k}> is in "references" but never described in the prompt; remove it or describe it.`)
|
||||
}
|
||||
}
|
||||
if (soundtrackLabels.length > 0 && problems.some((p) => p.includes('<Audio'))) {
|
||||
problems.push(`Audio labels in this job: ${soundtrackLabels.join(', ')} come from reference videos with sound, so audio files are numbered after them in list order.`)
|
||||
}
|
||||
|
||||
// Summary tag. A job whose only audio is a video's soundtrack may use either.
|
||||
const summary = positions[1] !== -1 ? lines[positions[1] + 1] ?? '' : ''
|
||||
const hasAudioTag = summary.includes('+ audio reference]')
|
||||
if (summary && audioFiles > 0 && !hasAudioTag) {
|
||||
problems.push('The summary tag must include "+ audio reference]" because the job has audio references.')
|
||||
} else if (summary && counts.Audio === 0 && hasAudioTag) {
|
||||
problems.push('The summary tag must not include "audio reference" because the job has no audio references.')
|
||||
}
|
||||
|
||||
// Shots.
|
||||
const shotLines = lines.filter((l) => /^\[Shot \d+\]/.test(l))
|
||||
if (shotLines.length === 0) problems.push('"detailed_description" needs at least one "[Shot 1] ..." line.')
|
||||
shotLines.forEach((line, i) => {
|
||||
const n = Number(line.match(/^\[Shot (\d+)\]/)![1])
|
||||
if (n !== i + 1) problems.push(`Shots must be numbered 1, 2, 3... in order; found [Shot ${n}] in position ${i + 1}.`)
|
||||
})
|
||||
let lastCut = 0
|
||||
for (const line of shotLines.slice(1)) {
|
||||
const cut = line.match(/^\[Shot (\d+)\] At (\d{2}:\d{2}\.\d{3}), the shot cuts to/)
|
||||
if (!cut) {
|
||||
problems.push(`[Shot ${line.match(/^\[Shot (\d+)\]/)![1]}] must start with "At MM:SS.mmm, the shot cuts to ...".`)
|
||||
continue
|
||||
}
|
||||
const t = seconds(cut[2])
|
||||
if (t <= lastCut) problems.push(`[Shot ${cut[1]}] cuts at ${cut[2]}, which is not after the previous cut.`)
|
||||
if (t >= duration) problems.push(`[Shot ${cut[1]}] cuts at ${cut[2]}, but the clip is only ${duration.toFixed(2)} s long (${frames} frames).`)
|
||||
lastCut = t
|
||||
}
|
||||
|
||||
// Appearance lists must name existing shots.
|
||||
for (const line of lines) {
|
||||
const m = line.match(/^<Subject (\d+)> \(appears in ([^)]*)\)/)
|
||||
if (!m) continue
|
||||
for (const ref of m[2].matchAll(/\[Shot (\d+)\]/g)) {
|
||||
if (Number(ref[1]) > shotLines.length) {
|
||||
problems.push(`<Subject ${m[1]}> is listed as appearing in [Shot ${ref[1]}], but there are only ${shotLines.length} shot(s).`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Dialogue tags.
|
||||
const opens = (text.match(/<d>/g) ?? []).length
|
||||
const closes = (text.match(/<\/d>/g) ?? []).length
|
||||
if (opens !== closes) problems.push(`Unbalanced dialogue tags: ${opens} "<d>" vs ${closes} "</d>".`)
|
||||
for (const m of text.matchAll(/<d>(?!\[English\] )/g)) {
|
||||
problems.push(`Every dialogue tag must start with "<d>[English] " (found one at character ${m.index}).`)
|
||||
break
|
||||
}
|
||||
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""API routes for the scene writer: start a draft (or a clip revision), then poll it.
|
||||
|
||||
A draft runs a DeepSeek Harness session for minutes, so each one runs on its
|
||||
own thread and the browser polls `GET /api/scene-writer/drafts/{run_id}`.
|
||||
Drafts never create studio jobs; the browser does that after review.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from fastvideo_studio.models import ClipRevisionRequest, SceneDraftRequest
|
||||
from fastvideo_studio.scene_writer.runner import (ReferenceFile, SceneWriterSettings, draft_scene, fetched_references,
|
||||
job_file_form, revise_clip)
|
||||
|
||||
logger = logging.getLogger("fastvideo.studio.scene_writer")
|
||||
|
||||
router = APIRouter(prefix="/api/scene-writer")
|
||||
|
||||
_drafts: dict[str, dict[str, Any]] = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _run(run_id: str, req: SceneDraftRequest, settings: SceneWriterSettings) -> None:
|
||||
try:
|
||||
result = draft_scene(
|
||||
script=req.script,
|
||||
references=[ReferenceFile(r.path, r.media_type, r.description) for r in req.references],
|
||||
scene_name=req.scene_name,
|
||||
instructions=req.instructions,
|
||||
settings=settings,
|
||||
run_id=run_id,
|
||||
find_references=req.find_references,
|
||||
)
|
||||
update = {
|
||||
"status": "completed",
|
||||
"jobs": result.jobs,
|
||||
"job_files": result.job_files,
|
||||
"final_response": result.final_response,
|
||||
"finish_reason": result.finish_reason,
|
||||
}
|
||||
except Exception as e: # surfaced to the browser; the run directory keeps the details
|
||||
logger.exception("Scene draft %s failed", run_id)
|
||||
update = {"status": "failed", "error": f"{type(e).__name__}: {e}"}
|
||||
with _lock:
|
||||
_drafts[run_id].update(update, finished_at=time.time())
|
||||
|
||||
|
||||
@router.post("/drafts", status_code=202)
|
||||
def start_draft(req: SceneDraftRequest) -> dict[str, Any]:
|
||||
settings = SceneWriterSettings()
|
||||
run_id = uuid.uuid4().hex[:12]
|
||||
with _lock:
|
||||
_drafts[run_id] = {
|
||||
"run_id": run_id,
|
||||
"scene_name": req.scene_name,
|
||||
"status": "running",
|
||||
"started_at": time.time(),
|
||||
"finished_at": None,
|
||||
"jobs": [],
|
||||
"error": None,
|
||||
"run_dir": str(settings.runs_root / run_id),
|
||||
}
|
||||
threading.Thread(target=_run, args=(run_id, req, settings), daemon=True,
|
||||
name=f"scene-draft-{run_id}").start()
|
||||
return get_draft(run_id)
|
||||
|
||||
|
||||
@router.get("/drafts/{run_id}")
|
||||
def get_draft(run_id: str) -> dict[str, Any]:
|
||||
with _lock:
|
||||
draft = _drafts.get(run_id)
|
||||
if draft is None:
|
||||
raise HTTPException(status_code=404, detail="Draft not found")
|
||||
draft = dict(draft)
|
||||
run_dir = Path(draft.pop("run_dir"))
|
||||
if draft["status"] == "running":
|
||||
# Show clips as the agent writes them; each file already passed validation.
|
||||
draft["jobs"] = [json.loads(f.read_text()) for f in sorted((run_dir / "jobs").glob("*.json"))]
|
||||
# Each image is served by name at /drafts/{run_id}/references/{file name}.
|
||||
draft["fetched_references"] = fetched_references(run_dir)
|
||||
return draft
|
||||
|
||||
|
||||
@router.get("/drafts/{run_id}/references/{name}")
|
||||
def get_fetched_reference(run_id: str, name: str) -> FileResponse:
|
||||
"""One image the writer downloaded for a draft, for review in the browser."""
|
||||
with _lock:
|
||||
draft = _drafts.get(run_id)
|
||||
if draft is None:
|
||||
raise HTTPException(status_code=404, detail="Draft not found")
|
||||
refs_dir = Path(draft["run_dir"]) / "refs"
|
||||
if name not in {Path(image["path"]).name for image in fetched_references(refs_dir.parent)}:
|
||||
raise HTTPException(status_code=404, detail="No such fetched reference")
|
||||
return FileResponse(refs_dir / name)
|
||||
|
||||
|
||||
# --- Revisions ----------------------------------------------------------------
|
||||
|
||||
_revisions: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _run_revision(run_id: str, req: ClipRevisionRequest, settings: SceneWriterSettings) -> None:
|
||||
try:
|
||||
result = revise_clip(req.job, req.feedback, req.neighbors, settings=settings, run_id=run_id)
|
||||
if result.job is None:
|
||||
update = {"status": "failed", "final_response": result.final_response,
|
||||
"error": "The reviser finished without writing a revised job."}
|
||||
else:
|
||||
update = {"status": "completed", "job": result.job, "final_response": result.final_response}
|
||||
except Exception as e: # surfaced to the browser; the run directory keeps the details
|
||||
logger.exception("Clip revision %s failed", run_id)
|
||||
update = {"status": "failed", "error": f"{type(e).__name__}: {e}"}
|
||||
with _lock:
|
||||
_revisions[run_id].update(update, finished_at=time.time())
|
||||
|
||||
|
||||
@router.post("/revisions", status_code=202)
|
||||
def start_revision(req: ClipRevisionRequest) -> dict[str, Any]:
|
||||
if not req.job.get("name"):
|
||||
raise HTTPException(status_code=422, detail="The job to revise has no name.")
|
||||
settings = SceneWriterSettings()
|
||||
run_id = uuid.uuid4().hex[:12]
|
||||
with _lock:
|
||||
_revisions[run_id] = {
|
||||
"run_id": run_id,
|
||||
"status": "running",
|
||||
"started_at": time.time(),
|
||||
"finished_at": None,
|
||||
"original": job_file_form(req.job),
|
||||
"job": None,
|
||||
"final_response": None,
|
||||
"error": None,
|
||||
}
|
||||
threading.Thread(target=_run_revision, args=(run_id, req, settings), daemon=True,
|
||||
name=f"clip-revision-{run_id}").start()
|
||||
return get_revision(run_id)
|
||||
|
||||
|
||||
@router.get("/revisions/{run_id}")
|
||||
def get_revision(run_id: str) -> dict[str, Any]:
|
||||
with _lock:
|
||||
revision = _revisions.get(run_id)
|
||||
if revision is None:
|
||||
raise HTTPException(status_code=404, detail="Revision not found")
|
||||
return dict(revision)
|
||||
@@ -0,0 +1,543 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Draft MiniMax-H3 jobs for a scene with a DeepSeek Harness agent.
|
||||
|
||||
One draft is one DSH SDK session in its own workspace:
|
||||
|
||||
<runs_root>/<run_id>/
|
||||
workspace/script.md the scene, as given
|
||||
workspace/references.md the reference files and what they are
|
||||
workspace/.dsh/skills/h3-job-writer/ the job-writing skill (copied)
|
||||
jobs/ the jobs the agent wrote and the validator passed
|
||||
scene-writer.patch.yml points the h3-scene-writer plugin at this run
|
||||
|
||||
The plugin is a DSH bundle (`dsh/`) installed into the harness home's `sdk`
|
||||
profile, so its harness imports resolve; it is (re)installed automatically
|
||||
whenever its files change.
|
||||
|
||||
The agent writes jobs only through the plugin's `h3_write_job` tool, which
|
||||
validates each one, so everything in `jobs/` already passed the structural
|
||||
checks. Nothing is posted to the studio here; the caller shows the drafts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
SKILLS_DIR = HERE / "skills"
|
||||
SKILL_DIR = SKILLS_DIR / "h3-job-writer"
|
||||
REVISER_SKILL_DIR = SKILLS_DIR / "h3-job-reviser"
|
||||
BUNDLE_DIR = HERE / "dsh"
|
||||
BUNDLE_SOURCES = ("h3-scene-writer.ts", "validate-job.ts", "reference-images.ts")
|
||||
BUNDLE_FILES = ("package.json", "cordis.patch.yml", *BUNDLE_SOURCES)
|
||||
|
||||
_install_lock = threading.Lock()
|
||||
|
||||
# The harness's own DeepSeek adapter; any other provider goes through its pi-ai
|
||||
# adapter (Cerebras, Groq, OpenAI, ...), which needs a route in the run's patch.
|
||||
DEEPSEEK_PROVIDER = "deepseek-official"
|
||||
DEFAULT_MODELS = {DEEPSEEK_PROVIDER: "deepseek-v4-flash", "cerebras": "qwen-3.8-27b"}
|
||||
# Models a provider serves that the harness's installed catalog does not list yet, as llm-pi-ai model profiles
|
||||
# (from the provider's own model list). `input` must name `image` for the agent's `read_image` tool to work.
|
||||
EXTRA_MODELS = {
|
||||
"cerebras": {
|
||||
"qwen-3.8-27b": {
|
||||
"name": "Qwen 3.8 27B",
|
||||
"contextWindow": 65536,
|
||||
"maxTokens": 32768,
|
||||
"input": ["text", "image"]
|
||||
},
|
||||
},
|
||||
}
|
||||
IMAGE_PIXEL_BUDGET = 1024 * 1024
|
||||
# The child env var the pi-ai route reads its key from.
|
||||
PI_AI_KEY_ENV = "SCENE_WRITER_API_KEY"
|
||||
|
||||
DEFAULT_SETTINGS_FILE = Path.home() / ".config" / "fastvideo-studio" / "scene-writer.env"
|
||||
|
||||
# Drafts hold the reference images their jobs point at, so they belong where
|
||||
# the server that renders those jobs can read them: beside its uploads.
|
||||
_server_runs_root: Path | None = None
|
||||
|
||||
|
||||
def use_server_data_dir(data_dir: Path) -> None:
|
||||
"""Keep drafts under the studio server's data dir unless SCENE_WRITER_RUNS_ROOT says otherwise."""
|
||||
global _server_runs_root
|
||||
_server_runs_root = Path(data_dir) / "scene_writer"
|
||||
|
||||
|
||||
def read_settings_file(path: Path) -> dict[str, str]:
|
||||
"""`KEY=value` lines from the settings file (an optional `export ` prefix and quotes allowed)."""
|
||||
values: dict[str, str] = {}
|
||||
if not path.is_file():
|
||||
return values
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.removeprefix("export ").split("=", 1)
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
||||
value = value[1:-1]
|
||||
values[key.strip()] = value
|
||||
return values
|
||||
|
||||
|
||||
def _setting(name: str, default=None):
|
||||
"""An environment variable, else the settings file, else the default. Read per draft, so file edits apply without a restart."""
|
||||
if name in os.environ:
|
||||
return os.environ[name]
|
||||
path = Path(os.environ.get("SCENE_WRITER_SETTINGS_FILE", DEFAULT_SETTINGS_FILE)).expanduser()
|
||||
return read_settings_file(path).get(name, default)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SceneWriterSettings:
|
||||
"""Where the harness lives and which model drives it (from the environment or the settings file by default)."""
|
||||
|
||||
harness_root: Path = field(
|
||||
default_factory=lambda: Path(_setting("SCENE_WRITER_HARNESS_ROOT", "/mnt/lustre/vlm-kevin/deepseek-harness")))
|
||||
runs_root: Path = field(default_factory=lambda: Path(
|
||||
_setting("SCENE_WRITER_RUNS_ROOT") or _server_runs_root or Path.home() / "h3_scene_writer").expanduser())
|
||||
node_bin_dir: str | None = field(default_factory=lambda: _setting("SCENE_WRITER_NODE_BIN_DIR"))
|
||||
provider: str = field(default_factory=lambda: _setting("SCENE_WRITER_PROVIDER", DEEPSEEK_PROVIDER))
|
||||
model: str | None = field(default_factory=lambda: _setting("SCENE_WRITER_MODEL"))
|
||||
base_url: str | None = field(default_factory=lambda: _setting("SCENE_WRITER_BASE_URL"))
|
||||
api_key: str | None = field(default_factory=lambda: _setting("SCENE_WRITER_API_KEY"))
|
||||
|
||||
@property
|
||||
def dsh_bin(self) -> Path:
|
||||
return self.harness_root / "apps" / "cli" / "lib" / "bin.js"
|
||||
|
||||
@property
|
||||
def sdk_src(self) -> Path:
|
||||
return self.harness_root / "python" / "sdk" / "src"
|
||||
|
||||
@property
|
||||
def dsh_home(self) -> Path:
|
||||
return self.runs_root / "dsh-home"
|
||||
|
||||
@property
|
||||
def uses_pi_ai(self) -> bool:
|
||||
return self.provider != DEEPSEEK_PROVIDER
|
||||
|
||||
def resolved_model(self) -> str:
|
||||
model = self.model or DEFAULT_MODELS.get(self.provider)
|
||||
if not model:
|
||||
raise RuntimeError(f"Set SCENE_WRITER_MODEL: there is no default model for provider {self.provider!r}.")
|
||||
return model
|
||||
|
||||
def child_env(self) -> dict[str, str]:
|
||||
env: dict[str, str] = {}
|
||||
if self.node_bin_dir:
|
||||
env["PATH"] = f"{self.node_bin_dir}{os.pathsep}{os.environ.get('PATH', '')}"
|
||||
if self.uses_pi_ai and self.api_key:
|
||||
env[PI_AI_KEY_ENV] = self.api_key
|
||||
return env
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReferenceFile:
|
||||
path: str
|
||||
media_type: str # "image" | "audio" | "video"
|
||||
description: str = ""
|
||||
|
||||
|
||||
def video_has_sound(path: str) -> bool:
|
||||
"""Whether a reference video carries a soundtrack; H3 then gives it an <Audio N> label too."""
|
||||
try:
|
||||
import av # noqa: PLC0415
|
||||
with av.open(path) as container:
|
||||
return bool(container.streams.audio)
|
||||
except Exception: # noqa: BLE001 - unreadable here means the validator's own existence check reports it
|
||||
return False
|
||||
|
||||
|
||||
def extract_frames(video: str,
|
||||
out_dir: Path,
|
||||
seconds_apart: float = 2.0,
|
||||
max_frames: int = 8,
|
||||
width: int = 768) -> list[Path]:
|
||||
"""Stills from a rendered take for the agent to look at: evenly spaced, about `seconds_apart` apart, each
|
||||
named by its time (`t03.5s.jpg`). Returns nothing for a missing or unreadable video."""
|
||||
try:
|
||||
import av # noqa: PLC0415
|
||||
with av.open(video) as container:
|
||||
stream = container.streams.video[0]
|
||||
frames = [(f.time or 0.0, f.to_image()) for f in container.decode(stream)]
|
||||
except Exception: # noqa: BLE001 - a take without a readable video is revised from its job alone
|
||||
return []
|
||||
if not frames:
|
||||
return []
|
||||
duration = frames[-1][0] + 1 / float(stream.average_rate or 24)
|
||||
count = max(1, min(max_frames, round(duration / seconds_apart)))
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
paths = []
|
||||
for i in range(count):
|
||||
# The middle of each equal slice, so the first and last stills are not the clip's edge frames.
|
||||
t, image = frames[min(len(frames) - 1, int((i + 0.5) * len(frames) / count))]
|
||||
image = image.resize((width, round(image.height * width / image.width)))
|
||||
path = out_dir / f"t{t:04.1f}s.jpg"
|
||||
image.save(path, quality=90)
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
@dataclass
|
||||
class DraftResult:
|
||||
run_id: str
|
||||
jobs: list[dict]
|
||||
job_files: list[str]
|
||||
final_response: str | None
|
||||
finish_reason: str | None
|
||||
fetched_references: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
def _import_sdk(settings: SceneWriterSettings):
|
||||
if str(settings.sdk_src) not in sys.path:
|
||||
sys.path.insert(0, str(settings.sdk_src))
|
||||
from deepseek_harness import DeepSeekHarness # noqa: PLC0415
|
||||
|
||||
return DeepSeekHarness
|
||||
|
||||
|
||||
def _yaml_str(value: str) -> str:
|
||||
return json.dumps(value) # JSON strings are valid YAML scalars
|
||||
|
||||
|
||||
def _bundle_digest() -> str:
|
||||
digest = hashlib.sha256()
|
||||
for name in BUNDLE_FILES:
|
||||
digest.update(name.encode())
|
||||
digest.update((BUNDLE_DIR / name).read_bytes())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _build_bundle(settings: SceneWriterSettings, env: dict[str, str]) -> None:
|
||||
"""Compile the plugin into one `dist/` file: DSH loads installed bundles as plain JavaScript."""
|
||||
esbuilds = sorted(
|
||||
(settings.harness_root / "node_modules" / ".pnpm").glob("esbuild@*/node_modules/esbuild/bin/esbuild"))
|
||||
if not esbuilds:
|
||||
raise RuntimeError(f"esbuild not found under {settings.harness_root}/node_modules; run `pnpm install` there.")
|
||||
subprocess.run([
|
||||
str(esbuilds[-1]), BUNDLE_SOURCES[0], "--bundle", "--external:@deepseek-ai/*", "--format=esm",
|
||||
"--platform=node", "--target=node22", "--outfile=dist/h3-scene-writer.js", "--log-level=warning"
|
||||
],
|
||||
cwd=BUNDLE_DIR,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120)
|
||||
|
||||
|
||||
def ensure_bundle_installed(settings: SceneWriterSettings) -> None:
|
||||
"""Install the h3-scene-writer bundle into the sdk profile, or refresh a stale copy."""
|
||||
marker = settings.dsh_home / "h3-scene-writer.installed"
|
||||
digest = _bundle_digest()
|
||||
with _install_lock:
|
||||
if marker.is_file() and marker.read_text() == digest:
|
||||
return
|
||||
settings.dsh_home.mkdir(parents=True, exist_ok=True)
|
||||
env = {**os.environ, **settings.child_env(), "DSH_HOME": str(settings.dsh_home)}
|
||||
_build_bundle(settings, env)
|
||||
proc = subprocess.run(
|
||||
["node",
|
||||
str(settings.dsh_bin), "plugin", "--profile", "sdk", "add", "--force", f"file:{BUNDLE_DIR}"],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
check=False)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Installing the scene-writer bundle failed:\n{proc.stdout[-2000:]}{proc.stderr[-2000:]}")
|
||||
marker.write_text(digest)
|
||||
|
||||
|
||||
def _yaml_list(values: list[str]) -> str:
|
||||
return "".join(f"\n - {_yaml_str(v)}" for v in values) or " []"
|
||||
|
||||
|
||||
def _write_patch(path: Path,
|
||||
out_dir: Path,
|
||||
references: list[ReferenceFile],
|
||||
sound_videos: list[str],
|
||||
refs_dir: Path | None,
|
||||
settings: SceneWriterSettings,
|
||||
expected_name: str = "") -> None:
|
||||
# An id-targeted row replaces the installed row's whole config.
|
||||
text = ("- id: h3-scene-writer\n"
|
||||
" config:\n"
|
||||
f" outDir: {_yaml_str(str(out_dir))}\n"
|
||||
f" references:{_yaml_list([r.path for r in references])}\n"
|
||||
f" soundVideos:{_yaml_list(sound_videos)}\n"
|
||||
f" refsDir: {_yaml_str(str(refs_dir) if refs_dir else '')}\n"
|
||||
f" expectedName: {_yaml_str(expected_name)}\n")
|
||||
if settings.uses_pi_ai:
|
||||
# The base profile mounts llm-pi-ai with no routes; this gives it one for the provider.
|
||||
text += ("- id: llm-pi-ai\n"
|
||||
" config:\n"
|
||||
" providers:\n"
|
||||
f" {_yaml_str(settings.provider)}:\n"
|
||||
f" apiKeyEnv: {PI_AI_KEY_ENV}\n" +
|
||||
(f" baseURL: {_yaml_str(settings.base_url)}\n" if settings.base_url else ""))
|
||||
model = settings.resolved_model()
|
||||
extra = EXTRA_MODELS.get(settings.provider, {}).get(model)
|
||||
if extra:
|
||||
# A `models` list replaces the route's catalog; the run only needs its own model.
|
||||
text += f" models: [{json.dumps({'id': model, **extra})}]\n"
|
||||
# Images the agent reads are sent at most this many pixels (the harness default is 2048x2048): enough to judge
|
||||
# a reference or a take, at a quarter of the tokens, which matters in a 65k context.
|
||||
text += f" requestImagePixelBudget: {IMAGE_PIXEL_BUDGET}\n"
|
||||
path.write_text(text)
|
||||
|
||||
|
||||
def _references_md(references: list[ReferenceFile], sound_videos: list[str], find_references: bool) -> str:
|
||||
|
||||
def row(r: ReferenceFile) -> str:
|
||||
kind = "video with sound: it also takes the next <Audio N> label" if r.path in sound_videos else r.media_type
|
||||
return f"- `{r.path}` ({kind}){': ' + r.description if r.description else ''}"
|
||||
|
||||
rows = "\n".join(row(r) for r in references) if references else "(none given)"
|
||||
text = f"# Reference files\n\nUse these exact absolute paths in `references`.\n\n{rows}\n"
|
||||
if find_references:
|
||||
text += ("\nYou may also use `find_reference_image` to download a photo for a person, place or object the "
|
||||
"script needs that none of these files shows. Never replace a file listed here.\n")
|
||||
return text
|
||||
|
||||
|
||||
def fetched_references(run_dir: Path) -> list[dict]:
|
||||
"""The images `find_reference_image` downloaded in a run, with their source pages and licenses."""
|
||||
sources = run_dir / "refs" / "sources.json"
|
||||
return json.loads(sources.read_text()) if sources.is_file() else []
|
||||
|
||||
|
||||
def build_prompt(scene_name: str, instructions: str) -> str:
|
||||
stem = scene_name.replace("-", "_")
|
||||
extra = f"\n\nAdditional direction from the user:\n{instructions.strip()}" if instructions.strip() else ""
|
||||
return ("/h3-job-writer\n\n"
|
||||
"Write MiniMax-H3 jobs for the scene in `script.md`, using only the reference files listed in "
|
||||
"`references.md` (and any it lets you download). "
|
||||
f"Name the jobs `{scene_name}-clip-01`, `{scene_name}-clip-02`, ... and pass `file_name` values "
|
||||
f"`{stem}_clip_01`, `{stem}_clip_02`, ... to `h3_write_job`. "
|
||||
"Write every clip through `h3_write_job` and fix anything it reports. Do not post or start jobs."
|
||||
f"{extra}")
|
||||
|
||||
|
||||
def _check_ready(settings: SceneWriterSettings) -> str:
|
||||
"""The model to use; raises when the harness or the model settings are not usable."""
|
||||
model = settings.resolved_model()
|
||||
if settings.uses_pi_ai and not settings.api_key:
|
||||
raise RuntimeError(f"Set SCENE_WRITER_API_KEY to your {settings.provider} API key.")
|
||||
if not settings.dsh_bin.is_file():
|
||||
raise RuntimeError(f"DeepSeek Harness is not built: {settings.dsh_bin} is missing "
|
||||
"(run `pnpm install && pnpm run build` in the harness checkout).")
|
||||
ensure_bundle_installed(settings)
|
||||
return model
|
||||
|
||||
|
||||
def _run_session(settings: SceneWriterSettings, model: str, workspace: Path, patch: Path, prompt: str, session_id: str,
|
||||
on_event: Callable[[dict], None] | None):
|
||||
"""Run one agent turn in `workspace` with the run's patch; returns the SDK's RunResult."""
|
||||
DeepSeekHarness = _import_sdk(settings)
|
||||
with DeepSeekHarness(
|
||||
dsh_home=str(settings.dsh_home),
|
||||
cwd=str(workspace),
|
||||
dsh_bin=str(settings.dsh_bin),
|
||||
profile="sdk",
|
||||
patches=(str(patch), ),
|
||||
provider=settings.provider,
|
||||
model=model,
|
||||
# These two reach only the DeepSeek adapter; a pi-ai route gets them from the patch.
|
||||
base_url=None if settings.uses_pi_ai else settings.base_url,
|
||||
api_key=None if settings.uses_pi_ai else settings.api_key,
|
||||
env=settings.child_env(),
|
||||
) as harness:
|
||||
result = harness.run(
|
||||
prompt,
|
||||
session_id=session_id,
|
||||
on_notification=(lambda n: on_event(n.payload)) if on_event else None,
|
||||
)
|
||||
# A plugin that fails to load only logs to stderr and the session runs without its tools.
|
||||
load_errors = [
|
||||
line for line in list(harness.client._stderr_lines)
|
||||
if line.startswith("h3-scene-writer (fastvideo-h3-scene-writer):")
|
||||
]
|
||||
if load_errors:
|
||||
raise RuntimeError(f"The scene-writer plugin failed to load: {load_errors[0]}")
|
||||
if result.finish_reason == "error":
|
||||
raise RuntimeError(_turn_error_message(result.events, settings))
|
||||
return result
|
||||
|
||||
|
||||
def _turn_error_message(events: list[dict], settings: SceneWriterSettings) -> str:
|
||||
"""Explain why the agent's turn ended in an error (usually the model provider refusing the request)."""
|
||||
message = "unknown error"
|
||||
for event in reversed(events):
|
||||
if event.get("type") == "turn/end":
|
||||
error = ((event.get("data") or {}).get("reason") or {}).get("error") or {}
|
||||
message = error.get("message") or message
|
||||
break
|
||||
if message.startswith("402"):
|
||||
return (f"{settings.provider} refused the request with 402 Payment Required: the account "
|
||||
"behind SCENE_WRITER_API_KEY has no credits or billing set up.")
|
||||
if message.startswith("401") or message.startswith("403"):
|
||||
return f"{settings.provider} rejected SCENE_WRITER_API_KEY ({message})."
|
||||
return f"The model call failed: {message}"
|
||||
|
||||
|
||||
def draft_scene(
|
||||
script: str,
|
||||
references: list[ReferenceFile],
|
||||
scene_name: str,
|
||||
instructions: str = "",
|
||||
settings: SceneWriterSettings | None = None,
|
||||
on_event: Callable[[dict], None] | None = None,
|
||||
run_id: str | None = None,
|
||||
find_references: bool = False,
|
||||
) -> DraftResult:
|
||||
"""Run one drafting session and return the jobs it wrote, in clip order.
|
||||
|
||||
With `find_references`, the agent may download reference photos from Wikimedia Commons into the run's
|
||||
`refs/` directory; `fetched_references` lists them.
|
||||
"""
|
||||
if not references and not find_references:
|
||||
raise ValueError("Give at least one reference file, or let the writer find references online.")
|
||||
settings = settings or SceneWriterSettings()
|
||||
model = _check_ready(settings)
|
||||
run_id = run_id or uuid.uuid4().hex[:12]
|
||||
run_dir = settings.runs_root / run_id
|
||||
workspace = run_dir / "workspace"
|
||||
out_dir = run_dir / "jobs"
|
||||
workspace.mkdir(parents=True, exist_ok=False)
|
||||
out_dir.mkdir()
|
||||
(workspace / "script.md").write_text(script)
|
||||
sound_videos = [r.path for r in references if r.media_type == "video" and video_has_sound(r.path)]
|
||||
(workspace / "references.md").write_text(_references_md(references, sound_videos, find_references))
|
||||
shutil.copytree(SKILL_DIR, workspace / ".dsh" / "skills" / SKILL_DIR.name)
|
||||
patch = run_dir / "scene-writer.patch.yml"
|
||||
_write_patch(patch, out_dir, references, sound_videos, run_dir / "refs" if find_references else None, settings)
|
||||
|
||||
result = _run_session(settings, model, workspace, patch, build_prompt(scene_name, instructions), f"scene-{run_id}",
|
||||
on_event)
|
||||
files = sorted(out_dir.glob("*.json"))
|
||||
return DraftResult(
|
||||
run_id=run_id,
|
||||
jobs=[json.loads(f.read_text()) for f in files],
|
||||
job_files=[str(f) for f in files],
|
||||
final_response=result.final_response,
|
||||
finish_reason=result.finish_reason,
|
||||
fetched_references=fetched_references(run_dir),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RevisionResult:
|
||||
run_id: str
|
||||
original: dict
|
||||
#: the revised job, or None when the agent wrote nothing
|
||||
job: dict | None
|
||||
#: the agent's explanation of the cause and its changes
|
||||
final_response: str | None
|
||||
finish_reason: str | None
|
||||
|
||||
|
||||
# The job-file fields; a Studio job also carries runtime state (status, outputs) the agent does not need.
|
||||
JOB_FILE_FIELDS = ("model_id", "name", "workload_type", "prompt", "references", "negative_prompt", "guidance_scale",
|
||||
"num_frames", "fps", "height", "width", "num_inference_steps", "seed", "num_gpus",
|
||||
"use_fsdp_inference")
|
||||
|
||||
|
||||
def job_file_form(job: dict) -> dict:
|
||||
"""A job in the job-file form the skills use: only job-file fields, `prompt` as a list of lines."""
|
||||
form = {k: job[k] for k in JOB_FILE_FIELDS if job.get(k) is not None}
|
||||
prompt = form.get("prompt", "")
|
||||
form["prompt"] = prompt.split("\n") if isinstance(prompt, str) else list(prompt)
|
||||
form["references"] = [{"source": r["source"], "media_type": r["media_type"]} for r in form.get("references") or []]
|
||||
return form
|
||||
|
||||
|
||||
def build_revision_prompt(job_name: str, file_name: str, has_frames: bool = False) -> str:
|
||||
frames = ("Stills from the rendered take are in `take_frames/`, named by their time in the clip; look at them "
|
||||
"with `read_image` and check the feedback against what was rendered. " if has_frames else "")
|
||||
return ("/h3-job-reviser\n\n"
|
||||
f"Revise the clip `{job_name}` in `current_job.json` to fix the problem described in `feedback.md`. "
|
||||
"The clips around it in the scene are in `neighbors/`. "
|
||||
f"{frames}"
|
||||
f"Write the revised job with `h3_write_job`, keeping the name `{job_name}` and passing `file_name` "
|
||||
f"`{file_name}`. Do not post or start jobs. Finish with the cause you found and each change you made.")
|
||||
|
||||
|
||||
def revise_clip(
|
||||
job: dict,
|
||||
feedback: str,
|
||||
neighbors: list[dict] | None = None,
|
||||
settings: SceneWriterSettings | None = None,
|
||||
on_event: Callable[[dict], None] | None = None,
|
||||
run_id: str | None = None,
|
||||
) -> RevisionResult:
|
||||
"""Run one revision session for a clip job (job-file or Studio form) and return the revised job.
|
||||
|
||||
The revision may use the clip's own references and its neighbours' (for example to add a voice that the
|
||||
clip before it already uses); it must keep the clip's name.
|
||||
"""
|
||||
name = job.get("name") or ""
|
||||
if not name:
|
||||
raise ValueError("The job to revise has no name.")
|
||||
if not feedback.strip():
|
||||
raise ValueError("Describe what is wrong with the clip.")
|
||||
settings = settings or SceneWriterSettings()
|
||||
model = _check_ready(settings)
|
||||
run_id = run_id or uuid.uuid4().hex[:12]
|
||||
run_dir = settings.runs_root / run_id
|
||||
workspace = run_dir / "workspace"
|
||||
out_dir = run_dir / "jobs"
|
||||
(workspace / "neighbors").mkdir(parents=True, exist_ok=False)
|
||||
out_dir.mkdir()
|
||||
|
||||
original = job_file_form(job)
|
||||
neighbors = [job_file_form(n) for n in neighbors or []]
|
||||
(workspace / "current_job.json").write_text(json.dumps(original, indent=2) + "\n")
|
||||
(workspace / "feedback.md").write_text(f"# What went wrong in {name}\n\n{feedback.strip()}\n")
|
||||
for n in neighbors:
|
||||
stem = (n.get("name") or "clip").replace("-", "_")
|
||||
(workspace / "neighbors" / f"{stem}.json").write_text(json.dumps(n, indent=2) + "\n")
|
||||
|
||||
references: list[ReferenceFile] = []
|
||||
for j in [original, *neighbors]:
|
||||
for r in j.get("references") or []:
|
||||
if r["source"] not in {ref.path for ref in references}:
|
||||
references.append(ReferenceFile(r["source"], r["media_type"]))
|
||||
sound_videos = [r.path for r in references if r.media_type == "video" and video_has_sound(r.path)]
|
||||
(workspace / "references.md").write_text(_references_md(references, sound_videos, False))
|
||||
for skill in (SKILL_DIR, REVISER_SKILL_DIR):
|
||||
shutil.copytree(skill, workspace / ".dsh" / "skills" / skill.name)
|
||||
patch = run_dir / "scene-writer.patch.yml"
|
||||
_write_patch(patch, out_dir, references, sound_videos, None, settings, expected_name=name)
|
||||
|
||||
# A Studio take carries its rendered video; a job file has none.
|
||||
frames = extract_frames(job["output_path"], workspace / "take_frames") if job.get("output_path") else []
|
||||
|
||||
file_name = name.replace("-", "_")
|
||||
result = _run_session(settings, model, workspace, patch, build_revision_prompt(name, file_name, bool(frames)),
|
||||
f"revise-{run_id}", on_event)
|
||||
# The newest file the agent wrote; the plugin already checked it keeps the clip's name.
|
||||
written = sorted(out_dir.glob("*.json"), key=lambda f: f.stat().st_mtime)
|
||||
return RevisionResult(
|
||||
run_id=run_id,
|
||||
original=original,
|
||||
job=json.loads(written[-1].read_text()) if written else None,
|
||||
final_response=result.final_response,
|
||||
finish_reason=result.finish_reason,
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
---
|
||||
name: h3-job-reviser
|
||||
description: Revise one MiniMax-H3 clip job from a person's feedback on its generated video, making the smallest prompt change that fixes the problem without breaking the clip's continuity with its neighbours.
|
||||
---
|
||||
|
||||
# Revising a MiniMax-H3 clip from feedback
|
||||
|
||||
A person generated a clip, watched it, and described what went wrong. You rewrite that clip's job
|
||||
so the next take fixes it. Work from the feedback, the job, what these rules say the model does,
|
||||
and, when `take_frames/` exists, stills from the take itself.
|
||||
|
||||
The job format, the section order, the numbering rules and the **Rules learned from real
|
||||
generations** are in `.dsh/skills/h3-job-writer/SKILL.md`. Read that file first; everything in it
|
||||
applies here.
|
||||
|
||||
## Inputs in the workspace
|
||||
|
||||
- `current_job.json`: the job as it was generated.
|
||||
- `feedback.md`: what went wrong, in the person's words.
|
||||
- `neighbors/`: the clips before and after it in the scene, when there are any. Positions,
|
||||
wardrobe, who is in the room and what the music is doing must stay consistent with them.
|
||||
- `references.md`: the reference files this job may use.
|
||||
- `take_frames/` (when the take's video was available): stills from the rendered take, about two
|
||||
seconds apart, named by their time in the clip (`t03.5s.jpg` is 3.5 s in).
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Read the writer skill, then the job, the feedback, and the neighbours. Look at every still in
|
||||
`take_frames/` and at the clip's image references with `read_image`. Use the stills to find
|
||||
where in the clip the problem starts and what the model actually rendered (an extra person,
|
||||
the wrong seat, a reference's background copied in, the wrong wardrobe). Stills are sparse and
|
||||
silent: for counts of fast repeated actions, timing within two seconds, and anything heard,
|
||||
trust the feedback over the stills. If the stills contradict the feedback, say so in your
|
||||
explanation and fix what the feedback describes.
|
||||
2. Diagnose before editing. Name the cause in the prompt, not just the symptom (table below).
|
||||
3. Make the smallest change that removes the cause. Keep everything that the feedback does not
|
||||
mention: wording that worked is worth keeping, and every change can create a new problem.
|
||||
4. Re-read the whole prompt for text the change made stale (rule 9 of the writer skill): the
|
||||
summary, the `appears in` lists, the soundscape, and any sentence that still describes the old
|
||||
behaviour. Fix those too.
|
||||
5. Write the job with `h3_write_job`, keeping its `name` and using the `file_name` you were given.
|
||||
Fix anything it reports.
|
||||
6. End with a short explanation for the person: the cause you found, each change you made, and
|
||||
anything they should watch for in the next take. If the feedback asks for something the
|
||||
format cannot promise, say so.
|
||||
|
||||
## Common symptoms and their usual causes
|
||||
|
||||
| Feedback | Usual cause | Fix |
|
||||
| --- | --- | --- |
|
||||
| An action happens too many times ("3 slaps instead of 2") | The prompt describes it as a repeated pattern ("on each four"), so the model keeps the rhythm going; or the clip has time left after the last scripted beat | Describe each occurrence once, tied to a word, with no "each"/"every"; remove counts like "exactly two"; end the clip shortly after the last one (shorten `num_frames`) |
|
||||
| A pause or stall at a line break | Speech split across several `<d>` tags with prose between them | Merge the run of speech into one `<d>` and put the actions after it |
|
||||
| Speech too slow or too fast | Ellipses, spelled-out pauses, or no tempo stated | Dashes for fast speech, no ellipses; state the tempo |
|
||||
| A character appears twice, or in the wrong place | The shot does not say where they are | Pin every character's position in every shot; add a whole-clip rule to the summary ("there is only one drummer") |
|
||||
| Someone moves too early | Their position is not pinned in the earlier shots, or the summary implies the move | Pin them in place in each shot before the move; move them only in the shot where the script does |
|
||||
| Background people vanish | A reference photo of an empty room, and the shot does not mention them | Name the background people in every shot, and say the photo shows the room before they arrived |
|
||||
| Music restarts or keeps playing | An audio reference plays from its start in every clip that uses it; the stop is not stated everywhere | Drop the music reference from clips after it ends; state when it stops in the summary, the shot and the soundscape |
|
||||
| Delivery is louder, angrier or more emphatic than wanted | Added direction ("roars", "shouts", all caps) | Remove it; keep the script's wording |
|
||||
| Something unwanted fills the end of the clip | The clip is longer than its scripted content | Shorten `num_frames` to end soon after the last beat |
|
||||
|
||||
When the feedback describes something new, reason the same way: what in the prompt (or missing
|
||||
from it) would make the model do that?
|
||||
|
||||
## Do not
|
||||
|
||||
- Do not change the clip's `name`, its references' order (labels like `<Picture 2>` depend on
|
||||
it), the resolution, or the seed, unless the feedback is about them.
|
||||
- Do not rewrite sections the feedback does not touch "while you are there".
|
||||
- Do not add a fix for a problem nobody reported.
|
||||
@@ -0,0 +1,190 @@
|
||||
---
|
||||
name: h3-job-writer
|
||||
description: Turn a screenplay scene plus reference images/audio into MiniMax-H3 Ref2VA job JSONs for FastVideo Studio, one job per clip, following the H3 prompt format and the timing/staging rules learned from real generations.
|
||||
---
|
||||
|
||||
# Writing MiniMax-H3 jobs from a script
|
||||
|
||||
You turn one scene of a script into a sequence of FastVideo Studio jobs, one per clip.
|
||||
Every clip is generated independently, so each job's prompt must stand on its own.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Read the script and the reference list you were given. Note every character, the setting,
|
||||
every action, and every line of dialogue in order. For the full prompt format with a complete
|
||||
example, read `.dsh/skills/h3-job-writer/official-guide.md` (MiniMax's own guide). Where it
|
||||
disagrees with this file, this file wins.
|
||||
Look at every image reference with `read_image` (see **Looking at references**).
|
||||
2. Plan the clips (see **Splitting into clips**). Write the plan down before drafting.
|
||||
3. Draft each clip as a job object and call `h3_write_job` for it. The tool validates the job
|
||||
and writes it; fix every error it reports and call it again until it succeeds.
|
||||
4. Finish with a short summary listing each clip: its file, length, and what happens in it.
|
||||
|
||||
Never post, queue, or start jobs. A human reviews every draft first.
|
||||
|
||||
## Job JSON
|
||||
|
||||
```json
|
||||
{
|
||||
"model_id": "MiniMaxAI/MiniMax-H3",
|
||||
"name": "<scene>-clip-01",
|
||||
"workload_type": "i2v",
|
||||
"prompt": ["subject_definitions:", "...", "", "summary:", "...", ...],
|
||||
"references": [
|
||||
{"source": "/abs/path/fletcher_ref.png", "media_type": "image"},
|
||||
{"source": "/abs/path/fletcher.mp3", "media_type": "audio"}
|
||||
],
|
||||
"negative_prompt": "",
|
||||
"guidance_scale": 1,
|
||||
"num_frames": 288,
|
||||
"fps": 24,
|
||||
"height": 768,
|
||||
"width": 1344,
|
||||
"num_inference_steps": 50,
|
||||
"seed": 0,
|
||||
"num_gpus": 4,
|
||||
"use_fsdp_inference": true
|
||||
}
|
||||
```
|
||||
|
||||
- `prompt` is a list of lines; they are joined with newlines before posting.
|
||||
- `num_frames` = clip seconds × 24, at most **345** (14.4 s). Size the clip to its scripted content (see rule 5).
|
||||
- Clip names share one stem and end in `-clip-NN` so Studio groups them into one scene.
|
||||
- Only use reference paths you were given. Include only the references a clip actually uses.
|
||||
- At most 9 images, 3 videos and 3 audio files per job, 12 references in total, and at least one
|
||||
image or video (audio alone is refused).
|
||||
|
||||
## Prompt sections (in this order)
|
||||
|
||||
```
|
||||
subject_definitions:
|
||||
<Subject 1> is <who> from <Picture 1>: <appearance, wardrobe, demeanor>.
|
||||
<Subject 2> is ...
|
||||
<Subject N> is the <location> from <Picture N>: <layout, fixtures>. It also sets the lighting for the whole scene: <light>. <What the picture is / is not, see rule 7>.
|
||||
<Audio 1> is the voice-timbre reference for <Subject 1> (S1), containing a spoken English vocal layer.
|
||||
<Video 1> is ... (only when continuing from a previous clip's tail)
|
||||
|
||||
summary:
|
||||
[reference generation + audio reference] The target video shows ... <global rules: who stays where, who never does what>. The setting is the <location> from <Subject N>, and the whole scene shares its <light>.
|
||||
|
||||
retention_analysis:
|
||||
<Subject 1> (appears in [Shot 1], [Shot 2]): fully_preserved - <what is kept>; his own background and lighting are not, and he is lit by the <location>'s light.
|
||||
<Subject N> (appears in [Shot 1], [Shot 2]): partially_preserved - <setting elements kept>; only the setting is retained, with no camera framing carried over.
|
||||
<Audio 1>: reference - its vocal timbre guides the dialogue delivery of <Subject 1> without copying the original signal.
|
||||
|
||||
detailed_description:
|
||||
[Shot 1] <Framing>. The camera is fixed and does not move. <Who is where>. <Action>. <Subject 1> (S1) speaks in the voice timbre referenced from <Audio 1>. (S1) <d>[English] Line.</d>
|
||||
[Shot 2] At 00:04.500, the shot cuts to <framing>. ...
|
||||
|
||||
overall_soundscape:
|
||||
<diegetic sound, with what must NOT be heard>
|
||||
|
||||
non_diegetic_music:
|
||||
N/A
|
||||
```
|
||||
|
||||
Numbering: `<Picture k>`, `<Video k>` and `<Audio k>` each count their own kind, in the order the
|
||||
references appear in `references`. A **video with sound** (marked in `references.md`) takes the
|
||||
next `<Audio k>` as well as its `<Video k>`, even if the prompt never mentions that soundtrack, so
|
||||
every audio file after it shifts up by one. Example: `[video with sound, voice.mp3, face.png]` are
|
||||
`<Video 1>` + `<Audio 1>`, then `<Audio 2>`, then `<Picture 1>`. (The official guide says a video
|
||||
does not create an audio label just because it has sound; FastVideo's pipeline does, and this rule
|
||||
follows the pipeline.) Subjects are numbered in definition order. The tag in the
|
||||
summary is `[reference generation + audio reference]` when any audio reference is used, else
|
||||
`[reference generation]`; add `video continuation + ` in front when a `<Video 1>` tail is used.
|
||||
|
||||
Dialogue: `(S1) <d>[English] text</d>`. Add `<cutoff>` after a line that is interrupted.
|
||||
Speaker IDs (S1, S2...) match the subject numbers of the speakers.
|
||||
|
||||
The `appears in` lists in `retention_analysis` must name exactly the shots where that subject is
|
||||
on screen. Keep them in sync whenever shots change.
|
||||
|
||||
## Rules learned from real generations
|
||||
|
||||
These are the reasons previous drafts failed. Follow all of them.
|
||||
|
||||
1. **Repeated actions become a rhythm the model continues.** "Slap on each four" turned into a
|
||||
steady beat that kept going. Keep a repeated action to the fewest repetitions the script needs,
|
||||
never write exceptions inside a pattern ("the third one gets no slap"), and end the pattern by
|
||||
ending the clip or cutting to a new shot. Counts like "exactly two" do not stop a rhythm.
|
||||
2. **Keep rhythmic speech in one dialogue tag.** Splitting a count into several `<d>` tags with
|
||||
prose between them makes the speaker pause at every break. Write one continuous line
|
||||
(`One-two-three-four-one-two-three-four-`) and describe actions that land on it in prose after it.
|
||||
3. **Write speech the way it should sound.** Ellipses slow delivery ("One... two..."); use dashes
|
||||
for fast speech. Spell words out ("four", not "fo-"). State tempo explicitly when it matters
|
||||
("150 BPM, 0.4 seconds per number, never pausing").
|
||||
4. **Timestamps are loose guides.** Use them for shot cuts and rough placement. Do not rely on a
|
||||
timestamp to sync an action to a word: anchor the action to the word itself ("as he says
|
||||
'four'"). Never pin an action to a time that the speech could drift away from.
|
||||
5. **Every second of the clip must be scripted.** Spare time at the end gets filled with more of
|
||||
whatever was happening (more slaps, the music starting over). End the clip shortly after its
|
||||
last scripted beat: estimate speech at ~2.5 words/second (faster for counting at the stated
|
||||
tempo) plus the time each action needs.
|
||||
6. **Pin every character's position in every shot.** If a shot does not say where someone is, the
|
||||
model invents it (a second copy of a character, someone walking over early). Put a whole-clip
|
||||
rule in the summary ("X stays seated behind his drum kit for the entire video; he never stands
|
||||
up", "there is only one drummer"), and restate positions in each shot. Use separate singles,
|
||||
not two-shots, when characters must stay apart. Frame anyone whose hands matter fully in shot;
|
||||
a half-framed actor's gestures read as extra actions.
|
||||
7. **Reference images win in wide shots.** An empty location photo produces an empty room. Say
|
||||
what the picture is ("This picture shows the room before the band arrives") and name the
|
||||
background people in every shot ("Every chair is filled: the band musicians sit at their
|
||||
stands..."), not just once in the subject definition.
|
||||
8. **Audio references play from their start.** A music reference restarts in every clip that uses
|
||||
it. Use music only in the clip it belongs to, and when music must stop, say when in the
|
||||
summary, the shot, and the soundscape ("the music stops dead at the crash; silence after").
|
||||
9. **Stale text keeps steering.** After changing a shot, re-read the summary, retention lists, and
|
||||
soundscape for anything that now contradicts it ("slapping him on each beat") and fix it.
|
||||
10. **Do not add direction the script does not ask for.** No added shouting, roaring, or
|
||||
emphasis unless the script says so; all-caps dialogue may be written in normal case only if
|
||||
the user asks. Keep the script's dialogue verbatim except for obvious typos you report.
|
||||
|
||||
## Finding missing references
|
||||
|
||||
Only when `references.md` says you may use `find_reference_image`:
|
||||
|
||||
- First decide which subjects need a reference: every named person on screen, the main location,
|
||||
and any object whose exact look matters. A subject already covered by a listed file keeps that
|
||||
file; never search for it or replace it.
|
||||
- Search for the real thing the script names: a real person or actor by full name ("Robert
|
||||
Pattinson"), a place by what it is ("jazz rehearsal room interior"). For a character an actor
|
||||
plays, search the actor only when the user's direction names the actor.
|
||||
- Call it once per subject with `count` 2-3, look at every result with `read_image`, and use
|
||||
exactly one image per subject: a clear photograph of only that subject (see **Looking at
|
||||
references** for what to reject). Add "photo" to a place's query ("diner interior photo").
|
||||
- If nothing usable comes back, try different words once, then write the clip without a reference
|
||||
for that subject and say so in your summary.
|
||||
- In your final summary, list each downloaded file you used and the subject it is for.
|
||||
|
||||
## Looking at references
|
||||
|
||||
Use `read_image` on every image reference, listed or downloaded, before you write a
|
||||
`subject_definitions` line for it. The model copies what is in the picture, not what the script
|
||||
says, so write from what you see:
|
||||
|
||||
- A person: apparent age, build, hair, and the clothing and accessories they wear in the picture.
|
||||
If that clothing does not suit the scene (a red-carpet gown for a diner), keep the face and
|
||||
build and dress them as the script needs: say in the definition that the picture supplies the
|
||||
face only, and mark them `partially_preserved` in `retention_analysis`.
|
||||
- A place: what is in it, the layout, the light and its color, and whether people are in it. Say
|
||||
that it supplies the setting only, and whether the scene's extras fill it or it stays empty.
|
||||
- Anything in the picture the clip must not copy: its background, its lighting, text, a frame.
|
||||
|
||||
Reject a downloaded image that is a painting, drawing, poster or screenshot when the scene is live
|
||||
action; that shows several people when you need one; that shows the wrong person or place; or
|
||||
that is dominated by text or a watermark. Never reject a listed file, but tell the person in your
|
||||
summary when it does not match the script (wrong era, wrong clothing, more than one person).
|
||||
|
||||
If `read_image` is refused because the model cannot take images, carry on without it: describe a
|
||||
downloaded subject only from its file title and the script, invent no details such as clothing,
|
||||
and say in your summary that the references were not checked.
|
||||
|
||||
## Splitting into clips
|
||||
|
||||
- 5–14.4 s per clip. Cut at natural beats: a change of action, a new exchange, a cut in the script.
|
||||
- The first clip of a scene usually establishes the setting and characters (e.g. the band playing).
|
||||
- Put each major physical action (a throw, a slap sequence) where the clip can end right after it.
|
||||
- Characters keep the same position across clips unless the script moves them, and a move happens
|
||||
in the clip where the script puts it, not earlier.
|
||||
- Fixed camera ("The camera is fixed and does not move.") in every shot unless the script asks
|
||||
for movement.
|
||||
@@ -0,0 +1,346 @@
|
||||
<!-- MiniMax-H3 full-reference prompt guide, copied unchanged from
|
||||
https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/42ed227ee7df40d41602854ae760620d6eb651fe/docs/VIDEO_PROMPT_WRITING_GUIDE_ref_en.md
|
||||
(via the dreamverse-harness video-harness branch). Where it disagrees with SKILL.md, SKILL.md wins:
|
||||
SKILL.md describes how FastVideo's pipeline actually labels references. -->
|
||||
|
||||
# Full-Reference Mode Rewrite Output Format Guide
|
||||
|
||||
This guide explains how rewrite outputs are organized and written in full-reference mode.
|
||||
|
||||
Write all six rewrite sections in English. Preserve the original language only for dialogue and lyrics inside `<d>` and for text visibly present in the scene.
|
||||
|
||||
**Description detail:** Make `detailed_description` as detailed and explicit as possible. For each shot, clearly establish the current composition, subject appearance and position, environment and lighting, actions and state changes, camera movement, current sound, and the points where referenced content actually appears or takes effect. Avoid reducing the description to a plot summary or a list of reference relationships.
|
||||
|
||||
> The basic formats for shots, camera movement, speakers, dialogue, and ordinary sound are shared with the Video Prompt Writing Guide (T2VA / I2VA / FL2VA / L2VA). This guide focuses on the reference labels, analysis sections, and format differences specific to full-reference mode.
|
||||
|
||||
## 1. Overall Structure
|
||||
|
||||
A complete rewrite output consists of six sections in the following order:
|
||||
|
||||
| Section | Purpose |
|
||||
| --- | --- |
|
||||
| `subject_definitions` | Defines referenced content and its reference labels |
|
||||
| `summary` | Summarizes the task type, target video, and main reference relationships |
|
||||
| `retention_analysis` | Describes how referenced content is preserved, transferred, or reused |
|
||||
| `detailed_description` | Describes visuals, actions, shots, sound, and dialogue in playback order |
|
||||
| `overall_soundscape` | Summarizes ambience and physical sounds |
|
||||
| `non_diegetic_music` | Describes background music audible only to the audience |
|
||||
|
||||
## 2. Reference Labels and Definitions (`subject_definitions`)
|
||||
|
||||
Full-reference rewrites use four types of labels to identify the source and role of referenced content:
|
||||
|
||||
| Label | Meaning |
|
||||
| --- | --- |
|
||||
| `<Subject N>` | Visible content abstracted from reference assets that can be reused or modified in the target video |
|
||||
| `<Picture N>` | A reference image used as a concrete target frame or shot-planning anchor |
|
||||
| `<Video N>` | A reference video that provides an editing source, continuation starting point, or whole-video temporal structure |
|
||||
| `<Audio N>` | An audio signal that is copied or referenced |
|
||||
|
||||
> Once a reference label is assigned to a piece of content, it keeps the same meaning across `subject_definitions`, `summary`, `retention_analysis`, `detailed_description`, and the audio sections.
|
||||
|
||||
`subject_definitions` defines each piece of referenced content that must be tracked separately later, such as a person, an environment, a source video's structure, or an audio track. Give each item its own line and explain what its label denotes, its reference role, and the main features to follow; name the corresponding source asset when its provenance needs to be made explicit. If `<Picture N>` or `<Video N>` only identifies the source of another referenced item and will not be analyzed or used separately later, cite it inside that item's definition without adding a separate line. `retention_analysis` records where each referenced item appears and whether it is fully preserved, partially preserved, transferred, or reused.
|
||||
|
||||
### 2.1 `<Subject N>`
|
||||
|
||||
`<Subject N>` is used for reusable visible content, including:
|
||||
|
||||
- People, animals, or objects
|
||||
- Scenes, backgrounds, or environments
|
||||
- Clothing, props, interfaces, or visual effects
|
||||
- Styles, actions, expressions, or poses
|
||||
|
||||
It represents a content unit that will actually be used in the target video, rather than the source file itself. One subject may be defined by multiple reference assets, and one reference asset may provide multiple subjects.
|
||||
|
||||
```text
|
||||
<Subject 1> is the young woman in <Picture 1>, with long dark hair, a blue cardigan, and a thin silver necklace.
|
||||
```
|
||||
|
||||
When the same subject comes from multiple assets, combine the sources and state what each asset provides:
|
||||
|
||||
```text
|
||||
<Subject 1> is the woman whose appearance comes from <Picture 1> and whose walking motion comes from <Video 1>.
|
||||
```
|
||||
|
||||
### 2.2 `<Picture N>`
|
||||
|
||||
Use a standalone `<Picture N>` when the reference image itself serves as a shot's first frame, keyframe, last frame, edited keyframe, or composition anchor:
|
||||
|
||||
```text
|
||||
<Picture 2> is the first frame of [Shot 1], showing a woman seated beside a café window.
|
||||
```
|
||||
|
||||
If an image is used only to define a character, scene, costume, or style, do not create a standalone picture entry. Instead, cite the image source inside the corresponding `<Subject N>` definition.
|
||||
|
||||
When an image acts as a storyboard or shot-planning reference, state which shots it maps to and what planning information it provides:
|
||||
|
||||
```text
|
||||
<Picture 3> is a storyboard reference for [Shot 1] and [Shot 2], defining their viewpoint, subject placement, and shot order.
|
||||
```
|
||||
|
||||
### 2.3 `<Video N>`
|
||||
|
||||
`<Video N>` is reserved for whole-video relationships, such as:
|
||||
|
||||
- Editing an original video
|
||||
- Continuing from the end of an original video
|
||||
- Referencing the original video's camera movement, cuts, rhythm, or temporal structure
|
||||
|
||||
```text
|
||||
<Video 1> is the source video for the target video edit.
|
||||
```
|
||||
|
||||
If a person, object, scene, action, or effect from a reference video is reused as visible content, it still belongs under `<Subject N>`. `<Video N>` identifies the asset or structural source and does not replace subject labels.
|
||||
|
||||
### 2.4 `<Audio N>`
|
||||
|
||||
`<Audio N>` represents a standalone audio asset or an enabled synchronized audio track from a reference video. Common uses include:
|
||||
|
||||
- Copying all or part of an audio signal
|
||||
- Referencing a background-music style
|
||||
- Referencing a speaker's voice timbre and delivery
|
||||
- Using dialogue, lyrics, or sound effects from the original audio
|
||||
- Referencing beat, rhythm, or audio continuity
|
||||
|
||||
When an `<Audio N>` explicitly corresponds to a target speaker, reuse that speaker's global ID in the definition: write `<Subject N> (Sx)` when the speaker maps to a defined subject, or use a stable voice description followed by `(Sx)` otherwise. The ID comes from the target video's global speaker order and is not independently assigned or renumbered in the audio definition. See Section 5.4 for the speaker-numbering rules:
|
||||
|
||||
```text
|
||||
<Audio 1> is the voice-timbre reference for <Subject 1> (S1).
|
||||
```
|
||||
|
||||
When one audio asset serves multiple roles, describe those roles in one natural sentence rather than creating additional subsections.
|
||||
|
||||
### 2.5 Visual and Audio Tracks from the Same Reference Video
|
||||
|
||||
`<Video N>` and `<Audio N>` are numbered independently. Each index indicates only the label's order within its own category and does not encode a pairing between the two categories. The same reference video may therefore correspond to `<Video 1>` and `<Audio 2>`; different indices do not prevent them from coming from the same source asset.
|
||||
|
||||
An ordinary reference video does not create `<Audio N>` merely because the file contains sound.
|
||||
|
||||
An `<Audio N>` definition primarily states the audio's role and does not have to name the `<Video N>` it comes from. State the shared source only when needed to remove provenance ambiguity, for example:
|
||||
|
||||
```text
|
||||
<Video 1> is the source video for the target video edit.
|
||||
<Audio 2> is the synchronized audio track of <Video 1> and is reused in the target video.
|
||||
```
|
||||
|
||||
## 3. `summary`
|
||||
|
||||
This section uses one short English paragraph to summarize the target video and its reference relationships. It begins with a square-bracketed task-type prefix:
|
||||
|
||||
```text
|
||||
[reference generation] ...
|
||||
[video editing + reference generation + audio reuse] ...
|
||||
```
|
||||
|
||||
Choose task types according to the actual role each reference asset plays in the target video:
|
||||
|
||||
| Task type | When to use it |
|
||||
| --- | --- |
|
||||
| `keyframe completion` | An image serves as the target video's first frame, keyframe, last frame, edited keyframe, or another concrete frame anchor |
|
||||
| `reference generation` | An image, video, or audio asset provides generation guidance for a character, scene, style, action, camera movement, storyboard, and so on, without serving as a concrete frame or as the source video being edited or continued |
|
||||
| `video editing` | An existing source video is directly modified; editing an image or generating between still keyframes does not belong to this type |
|
||||
| `video continuation` | New content continues, extends, resumes, or transitions from an existing source video |
|
||||
| `audio reuse` | The same audio signal is reused in full or in part |
|
||||
| `audio reference` | The audio signal is not copied directly; only its music style, timbre, dialogue or lyric content, sound-effect texture, beat, or continuity is referenced |
|
||||
|
||||
When a task satisfies multiple relationships, combine the task types with ` + ` and do not repeat a type. For example, continuing from a source video while using an image as the last frame is written as `[video continuation + keyframe completion]`. Editing a source video while retaining its original audio may be written as `[video editing + audio reuse]`.
|
||||
|
||||
The mere presence of video or audio does not automatically create a corresponding task type. If a reference video provides only camera movement, cuts, or rhythm, it normally belongs to `reference generation`. Use `video editing` or `video continuation` only when that video is directly edited or continued.
|
||||
|
||||
When editing a source video, use `audio reuse` as well if its original audio remains audible. When continuing a source video without directly copying the audio signal, use `audio reference` if the new audio only continues the original track's audible characteristics.
|
||||
|
||||
The summary uses the previously defined `<Subject N>`, `<Picture N>`, `<Video N>`, and `<Audio N>` labels to describe the main subjects, shot flow, and roles of the reference assets. Do not introduce new reference labels in this section.
|
||||
|
||||
For video-editing tasks, begin the summary after the task-type prefix with:
|
||||
|
||||
```text
|
||||
The target video is an edited version of <Video 1>.
|
||||
```
|
||||
|
||||
## 4. `retention_analysis`
|
||||
|
||||
This section describes how each piece of referenced content is preserved, transferred, copied, or referenced in the target video. Use one line for each reference label and preserve the meaning established in `subject_definitions`.
|
||||
|
||||
### 4.1 Visible Content
|
||||
|
||||
`<Subject N>`, `<Picture N>`, and `<Video N>` use the following relationship markers. These markers are fixed English values in the output format:
|
||||
|
||||
| Relationship marker | Meaning |
|
||||
| --- | --- |
|
||||
| `fully_preserved` | The defined role of the referenced content is fully preserved |
|
||||
| `partially_preserved` | The referenced content is still used, but some defined characteristics are changed or only partially retained |
|
||||
| `attribute_transfer` | Referenced characteristics are transferred to a different identifiable target subject |
|
||||
| `weak_reference` | Only broad similarity in style, category, composition, or atmosphere is retained |
|
||||
|
||||
Subject entry:
|
||||
|
||||
```text
|
||||
<Subject 1> (appears in [Shot 1], [Shot 3]): fully_preserved - ...
|
||||
```
|
||||
|
||||
Picture entry:
|
||||
|
||||
```text
|
||||
<Picture 2> ([Shot 1] first frame): fully_preserved - ...
|
||||
```
|
||||
|
||||
Video-structure entry:
|
||||
|
||||
```text
|
||||
<Video 1> (cut and pacing structure): weak_reference - ...
|
||||
```
|
||||
|
||||
### 4.2 Audio
|
||||
|
||||
`<Audio N>` uses the following relationship markers:
|
||||
|
||||
| Relationship marker | Meaning |
|
||||
| --- | --- |
|
||||
| `fully_copy` | The complete source audio serves as the target video's complete final audio track |
|
||||
| `partially_copy` | Only part of the timeline or selected audio layers are copied, or other sounds are added, removed, or replaced after copying |
|
||||
| `reference` | The signal is not copied directly; only timbre, rhythm, music style, dialogue content, or sound texture is referenced |
|
||||
| `weak_reference` | Only broad similarity in category or atmosphere is retained |
|
||||
|
||||
```text
|
||||
<Audio 1>: fully_copy - <Audio 1> is reused 1:1 as the target video's complete final audio track.
|
||||
```
|
||||
|
||||
```text
|
||||
<Audio 2>: reference - the target speaker follows <Audio 2>'s voice timbre and measured delivery without copying the original signal.
|
||||
```
|
||||
|
||||
Choose each relationship marker only within the reference role already defined for that label in `subject_definitions`. Do not treat newly added actions, backgrounds, or plot events in the target video as losses of reference fidelity.
|
||||
|
||||
## 5. `detailed_description`
|
||||
|
||||
This is the main body of a full-reference rewrite. It describes visuals, actions, sound, and dialogue shot by shot in target-video playback order and inserts reference labels where they apply.
|
||||
|
||||
### 5.1 Basic Format
|
||||
|
||||
The basic format follows the Video Prompt Writing Guide (T2VA / I2VA / FL2VA / L2VA):
|
||||
|
||||
- Write the body in English. Preserve the original language of dialogue, lyrics, and visible text.
|
||||
- `[Shot 1]` marks the opening shot and has no timestamp. Later shots use `[Shot N] At MM:SS.mmm, ...` to mark cut times.
|
||||
- Write camera movement as natural English within the current shot, including movement type, amplitude, and speed when they need to be expressed.
|
||||
- Give vocal sources stable `(S1)`, `(S2)`, and subsequent IDs. Write dialogue and lyrics as `<d>[Language] ...</d>`.
|
||||
- Use `<scenetrans>`, `<cutoff>`, and the corresponding continuity descriptions for dialogue crossing a cut, speech truncated by the video ending, and continuous audio across shots.
|
||||
|
||||
For complete rules and examples covering camera vocabulary, group speech, voice-over, dialogue across cuts, and visible text, see the Video Prompt Writing Guide (T2VA / I2VA / FL2VA / L2VA).
|
||||
|
||||
### 5.2 Full-Reference Mode Differences
|
||||
|
||||
| Dimension | T2VA | Full-reference mode |
|
||||
| --- | --- | --- |
|
||||
| Main field | `integrated_multimodal_description` | `detailed_description` |
|
||||
| Style opening | Written after `[Shot 1]` | Established in one or two English sentences before `[Shot 1]` |
|
||||
| Reference information | Does not use full-reference labels | Inserts `<Subject N>`, `<Picture N>`, `<Video N>`, and `<Audio N>` at their first appearance and where their roles apply |
|
||||
| Audio relationships | Describes the target video's own sound | Cites `<Audio N>` in the corresponding shot or audio phase and states whether the signal is copied or referenced |
|
||||
|
||||
Opening example:
|
||||
|
||||
```text
|
||||
The target video is in a cinematic, literary music-video style with soft lighting and a slightly desaturated color palette.
|
||||
[Shot 1] The scene opens in a crowded urban street...
|
||||
[Shot 2] At 00:09.000, the shot cuts to an extreme close-up...
|
||||
```
|
||||
|
||||
For generation tasks, `detailed_description` is normally 350-500 English words. Dialogue-dense content prioritizes fitting the complete spoken timeline rather than mechanically reaching a word count. Video-editing descriptions scale with the complexity of the source video and do not have to follow the generation-task range. A single shot does not automatically justify a shorter description; distribute detail across multiple shots according to their information load.
|
||||
|
||||
### 5.3 Using Reference Labels in Shots
|
||||
|
||||
At the first clear appearance of an important `<Subject N>`, describe its referenced characteristics, position in the frame, and current action within what is actually visible in the shot. Continue using the same label in later shots without redefining what the label represents.
|
||||
|
||||
Use natural phrasing for concrete frame anchors:
|
||||
|
||||
```text
|
||||
the shot begins from <Picture 1>
|
||||
the shot's keyframe corresponds to <Picture 2>
|
||||
the shot ends on <Picture 3>
|
||||
```
|
||||
|
||||
When editing or continuing an original video, cite `<Video N>` naturally where its source state, structure, or continuation relationship applies. Cite `<Audio N>` in the shot or semantic phase where the audio relationship is active.
|
||||
|
||||
### 5.4 Speakers, Audio Sources, and Dialogue
|
||||
|
||||
The basic speaker-ID and `<d>` formats follow T2VA. When a referenced subject physically speaks, retain both the visual reference label and the speaker ID:
|
||||
|
||||
```text
|
||||
<Subject 2> (S1) turns toward the woman and says, <d>[English] Last summer, I went to my grandfather's house. He talked about you.</d>
|
||||
```
|
||||
|
||||
`<Subject N>` identifies the referenced subject, while `(Sx)` identifies the actual speaker. When the subject speaks, write `<Subject N> (Sx)`. If the same subject speaks off-screen, keep the same form and mark it as `off-screen`. When the speaker does not correspond to a defined subject, use a stable voice description followed by `(Sx)`.
|
||||
|
||||
When verbal content is only a cue within a directly reused BGM or complete soundtrack, and no person, character, narrator, or other independent vocal source physically produces it, use `<Audio N>` as the audible source and do not invent an additional `(Sx)`. If a concrete person, character, narrator, or other independent vocal source produces the voice, assign and reuse `(Sx)` for that source:
|
||||
|
||||
```text
|
||||
When <Audio 1> reaches the phrase <d>[English] I'm lonely lonely lonely lonely lonely I'm lonely</d>, <Subject 1> performs the corresponding hand gesture without becoming a separate speaker source.
|
||||
```
|
||||
|
||||
When dialogue, narration, or lyrics from reference audio are directly reused, or when the input prompt explicitly requests their reperformance, preserve the exact source words and original language inside `<d>`. Write `[unclear]` for unintelligible spans instead of guessing or paraphrasing them. Standardize punctuation to the basic written marks needed to express the sentence, such as `,`, `.`, `?`, and `!`; remove repeated tildes, emoji, bullets, and repeated or decorative punctuation. End complete statements, questions, and exclamations with `.`, `?`, or `!` respectively before `</d>`.
|
||||
|
||||
When only timbre, rhythm, emotion, or delivery is referenced, do not carry the original dialogue from the reference audio into the target video.
|
||||
|
||||
Assign `(Sx)` once according to the order of actual vocal events in the target video. Reuse the corresponding ID at every actual vocal event in `detailed_description`; an `<Audio N>` definition bound to a target speaker in `subject_definitions` also reuses the same `(Sx)` but never assigns a new one independently. Do not write `(Sx)` in `retention_analysis`. Verbal cues that exist only within a directly reused BGM or complete soundtrack use `<Audio N>`; voices physically produced by a concrete person, character, narrator, or other independent vocal source use `(Sx)`.
|
||||
|
||||
## 6. `overall_soundscape` and `non_diegetic_music`
|
||||
|
||||
The definitions of these two sound categories follow the Video Prompt Writing Guide (T2VA / I2VA / FL2VA / L2VA).
|
||||
|
||||
`overall_soundscape` summarizes ambience and physical sounds across the full video. Dialogue, singing, and sound events synchronized to a particular shot remain in `detailed_description`:
|
||||
|
||||
```text
|
||||
overall_soundscape: Quiet indoor room tone and a low ventilation hum continue throughout the video.
|
||||
```
|
||||
|
||||
`non_diegetic_music` describes background music that the characters cannot hear and that is audible only to the audience. When music is present, state its instrumentation, tempo, and dynamic development:
|
||||
|
||||
```text
|
||||
non_diegetic_music: A restrained solo-piano score at a slow tempo, with sustained low cello underneath and no swell.
|
||||
```
|
||||
|
||||
When reference audio is used, state its copy or reference relationship only in the section that matches the audible layer: ambience and sound effects belong in `overall_soundscape`, while audience-only score belongs in `non_diegetic_music`. If the same audio provides both kinds of content, describe the corresponding relationship in each section:
|
||||
|
||||
```text
|
||||
overall_soundscape: The copied ambience layer from <Audio 1> continues throughout the target video.
|
||||
non_diegetic_music: <Audio 2> is directly reused as the complete audience-only score.
|
||||
```
|
||||
|
||||
Write complete dialogue and lyrics only inside `<d>` in `detailed_description`; do not repeat them in these two sections.
|
||||
|
||||
## 7. Complete Example
|
||||
|
||||
<details>
|
||||
<summary>Show the complete example</summary>
|
||||
|
||||
```text
|
||||
subject_definitions:
|
||||
<Subject 1> is the coffee-shop environment in <Picture 1>, featuring an exposed brick wall, an orange tufted sofa with patterned pillows, a neon sign, and a wooden coffee table.
|
||||
<Subject 2> is the fluffy white Samoyed in <Picture 2>, <Picture 3>, and <Picture 4>, with thick white fur, pointed ears, a dark nose, and a curved tail.
|
||||
<Subject 3> is the young blonde woman in <Video 1>, with long blonde hair and a light-pink button-down shirt with rolled-up sleeves.
|
||||
<Subject 4> is the young man in <Video 2>, with short wavy brown hair and a dark-grey hoodie with drawstrings.
|
||||
<Audio 1> is the voice-timbre reference for <Subject 3> (S1), containing a spoken English vocal layer.
|
||||
|
||||
summary:
|
||||
[reference generation + audio reference] The target video shows <Subject 3> eating a cookie in <Subject 1>. <Subject 4> enters with <Subject 2>, which lunges toward the cookie. The three-shot exchange uses <Audio 1> as the voice-timbre reference for <Subject 3> and ends with a canned audience laugh.
|
||||
|
||||
retention_analysis:
|
||||
<Subject 1> (appears in [Shot 1], [Shot 2], [Shot 3]): fully_preserved - the exposed brick wall, orange tufted sofa, patterned pillows, neon sign, and wooden coffee table are retained.
|
||||
<Subject 2> (appears in [Shot 1], [Shot 2]): fully_preserved - the Samoyed's thick white fur, pointed ears, dark nose, and curved tail are retained.
|
||||
<Subject 3> (appears in [Shot 1], [Shot 2], [Shot 3]): fully_preserved - the blonde woman's identity, long hair, and light-pink shirt are retained.
|
||||
<Subject 4> (appears in [Shot 1], [Shot 2]): fully_preserved - the young man's short wavy brown hair and dark-grey hoodie are retained.
|
||||
<Audio 1>: reference - its vocal timbre guides the dialogue delivery of <Subject 3> without copying the original signal.
|
||||
|
||||
detailed_description:
|
||||
The target video uses a realistic multi-camera sitcom style with warm indoor lighting.
|
||||
[Shot 1] A medium shot establishes <Subject 1>, the coffee shop with its exposed brick wall, orange tufted sofa, patterned pillows, neon sign, and wooden coffee table. <Subject 3> (S1), the young woman with long blonde hair and a light-pink button-down shirt with rolled-up sleeves, sits on the sofa holding a chocolate-chip cookie. From the left, <Subject 4>, the young man with short wavy brown hair and a dark-grey hoodie with drawstrings, enters holding the leash of <Subject 2>, the thick-furred white Samoyed with pointed ears, a dark nose, and a curved tail. The dog lunges toward the cookie and pulls the leash taut. <Subject 3> (S1) jerks her hand back and, using the clear youthful voice timbre referenced from <Audio 1>, exclaims with light annoyance, <d>[English] Hey! Watch your dog!</d> She closes her lips and guards the cookie while <Subject 4> pulls the dog back.
|
||||
[Shot 2] At 00:03.000, the shot cuts to a close-up of <Subject 4> (S2), the young man in the dark-grey hoodie from Shot 1, sitting beside <Subject 3> on the sofa and holding <Subject 2> securely in his arms. <Subject 4> (S2) says in a casual young male voice with a playful tone and an easy conversational pace, <d>[English] He just likes cookies more than me.</d> He closes his mouth into an apologetic smile and strokes the dog's thick white fur.
|
||||
[Shot 3] At 00:05.000, the shot cuts to a close-up of <Subject 3> (S1), the blonde woman in the light-pink shirt from Shot 1. Her annoyance softens as she looks toward the Samoyed. <Subject 3> (S1) replies in the same clear youthful voice referenced from <Audio 1> with an amused cadence, <d>[English] Well, he has good taste at least.</d> She smiles and raises the cookie in a small toast-like gesture. A classic canned audience laugh begins immediately after the line and continues through the final frame.
|
||||
|
||||
overall_soundscape:
|
||||
Soft indoor coffee-shop room tone continues throughout the scene.
|
||||
|
||||
non_diegetic_music:
|
||||
N/A
|
||||
```
|
||||
|
||||
</details>
|
||||
@@ -33,10 +33,15 @@ from fastapi.responses import FileResponse
|
||||
|
||||
from fastvideo.registry import (get_registered_model_paths, get_registered_models_with_workloads)
|
||||
from fastvideo_studio.database import Database, _get_db_path
|
||||
from fastvideo_studio.frames import FrameError, last_clip_for_job, last_frame_for_job
|
||||
from fastvideo_studio.gpu import get_gpu_snapshot
|
||||
from fastvideo_studio.job_runner import JobRunner, JobStatus
|
||||
from fastvideo_studio.models import (CreateDatasetRequest, CreateJobRequest, SettingsUpdate, UpdateCaptionRequest,
|
||||
model_label)
|
||||
from fastvideo_studio.merge import MergeError, merge_jobs, merged_path
|
||||
from fastvideo_studio.reference_preview import reference_image_response
|
||||
from fastvideo_studio.scene_writer.routes import router as scene_writer_router
|
||||
from fastvideo_studio.scene_writer.runner import use_server_data_dir
|
||||
from fastvideo_studio.models import (CreateDatasetRequest, CreateJobRequest, MergeSceneRequest, QueueJobsRequest,
|
||||
SettingsUpdate, TrimRequest, UpdateCaptionRequest, model_label)
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -70,6 +75,8 @@ app.add_middleware(
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(scene_writer_router)
|
||||
|
||||
|
||||
@app.get("/api/settings")
|
||||
def get_settings() -> dict[str, Any]:
|
||||
@@ -439,6 +446,41 @@ def start_job(job_id: str) -> dict[str, Any]:
|
||||
) from e
|
||||
|
||||
|
||||
def _queue_error(e: ValueError) -> HTTPException:
|
||||
return HTTPException(status_code=404 if "not found" in str(e) else 409, detail=str(e))
|
||||
|
||||
|
||||
@app.post("/api/jobs/queue")
|
||||
def queue_jobs(req: QueueJobsRequest) -> list[dict[str, Any]]:
|
||||
"""Queue jobs, in the order given, to run as GPU slots free up.
|
||||
|
||||
A job that starts from another job's last frame waits for that job to
|
||||
complete. All-or-nothing: if any job can't be queued, none are.
|
||||
"""
|
||||
try:
|
||||
return [j.to_dict() for j in job_runner.enqueue_jobs(req.job_ids)]
|
||||
except ValueError as e:
|
||||
raise _queue_error(e) from e
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/queue")
|
||||
def queue_job(job_id: str) -> dict[str, Any]:
|
||||
"""Queue one job to run when a slot is free."""
|
||||
try:
|
||||
return job_runner.enqueue_job(job_id).to_dict()
|
||||
except ValueError as e:
|
||||
raise _queue_error(e) from e
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/dequeue")
|
||||
def dequeue_job(job_id: str) -> dict[str, Any]:
|
||||
"""Take a queued job out of the queue (back to pending)."""
|
||||
try:
|
||||
return job_runner.dequeue_job(job_id).to_dict()
|
||||
except ValueError as e:
|
||||
raise _queue_error(e) from e
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/stop")
|
||||
def stop_job(job_id: str) -> dict[str, Any]:
|
||||
"""Request a running job to stop.
|
||||
@@ -665,6 +707,122 @@ def get_video(job_id: str) -> FileResponse:
|
||||
return FileResponse(job.output_path, media_type=media_type)
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/reference")
|
||||
def get_reference_image(job_id: str, source: str) -> FileResponse:
|
||||
"""One of the job's image references, for a thumbnail in the job form."""
|
||||
job = job_runner.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
return reference_image_response(job.references, source)
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/last-frame")
|
||||
def extract_last_frame(job_id: str) -> dict[str, str]:
|
||||
"""Save the last frame of a completed job's video and return its path.
|
||||
|
||||
The path is meant to be attached to another job as an image reference, so
|
||||
the next clip of a scene can start from where this one ended.
|
||||
"""
|
||||
job = job_runner.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
try:
|
||||
path = last_frame_for_job(job, upload_dir)
|
||||
except FrameError as e:
|
||||
if e.status_code >= 500:
|
||||
logger.warning("Last-frame extraction failed for job %s: %s", job_id, e.detail)
|
||||
raise HTTPException(status_code=e.status_code, detail=e.detail) from e
|
||||
return {"path": path, "media_type": "image"}
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/last-clip")
|
||||
def extract_last_clip(job_id: str) -> dict[str, str]:
|
||||
"""Save the trailing second of a completed job's video and return its path.
|
||||
|
||||
The path is meant to be attached to another job as a video reference, so
|
||||
the next clip continues with real motion instead of guessing it from a
|
||||
single still frame.
|
||||
"""
|
||||
job = job_runner.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail="Job not found")
|
||||
try:
|
||||
path = last_clip_for_job(job, upload_dir)
|
||||
except FrameError as e:
|
||||
if e.status_code >= 500:
|
||||
logger.warning("Last-clip extraction failed for job %s: %s", job_id, e.detail)
|
||||
raise HTTPException(status_code=e.status_code, detail=e.detail) from e
|
||||
return {"path": path, "media_type": "video"}
|
||||
|
||||
|
||||
def _trim_error(e: Exception) -> HTTPException:
|
||||
if isinstance(e, FrameError):
|
||||
return HTTPException(status_code=e.status_code, detail=e.detail)
|
||||
return HTTPException(status_code=404 if "not found" in str(e) else 400, detail=str(e))
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/trim")
|
||||
def trim_job(job_id: str, req: TrimRequest) -> dict[str, Any]:
|
||||
"""Cut a completed job's video to [start_seconds, end_seconds) and grade it section by section, in place.
|
||||
|
||||
The untouched original is kept, so this can be called again with a
|
||||
different range and/or sections, or undone with POST .../restore-video.
|
||||
"""
|
||||
try:
|
||||
segments = [s.model_dump() for s in req.segments]
|
||||
job = job_runner.trim_job(job_id, req.start_seconds, req.end_seconds, segments)
|
||||
except (ValueError, FrameError) as e:
|
||||
raise _trim_error(e) from e
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/restore-video")
|
||||
def restore_job_video(job_id: str) -> dict[str, Any]:
|
||||
"""Undo every trim on a job's video, back to what it originally generated."""
|
||||
try:
|
||||
job = job_runner.restore_job_video(job_id)
|
||||
except (ValueError, FrameError) as e:
|
||||
raise _trim_error(e) from e
|
||||
return job.to_dict()
|
||||
|
||||
|
||||
@app.post("/api/scenes/merge")
|
||||
def merge_scene(req: MergeSceneRequest) -> dict[str, Any]:
|
||||
"""Join the finished clips of a scene, in the order given, into one video.
|
||||
|
||||
The clips are copied, not re-encoded, so this takes about a second.
|
||||
"""
|
||||
jobs = []
|
||||
for job_id in req.job_ids:
|
||||
job = job_runner.get_job(job_id)
|
||||
if job is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
jobs.append(job)
|
||||
try:
|
||||
result = merge_jobs(jobs, job_runner.output_dir, req.name)
|
||||
except MergeError as e:
|
||||
if e.status_code >= 500:
|
||||
logger.warning("Scene merge failed: %s", e.detail)
|
||||
raise HTTPException(status_code=e.status_code, detail=e.detail) from e
|
||||
logger.info("Merged %d clips into %s", result.clips, result.path)
|
||||
return {
|
||||
"filename": result.filename,
|
||||
"url": f"/api/merged/{result.filename}",
|
||||
"clips": result.clips,
|
||||
"seconds": result.seconds,
|
||||
}
|
||||
|
||||
|
||||
@app.get("/api/merged/{filename}")
|
||||
def get_merged_video(filename: str) -> FileResponse:
|
||||
"""Stream a merged scene video."""
|
||||
try:
|
||||
path = merged_path(job_runner.output_dir, filename)
|
||||
except MergeError as e:
|
||||
raise HTTPException(status_code=e.status_code, detail=e.detail) from e
|
||||
return FileResponse(path, media_type="video/mp4", filename=filename)
|
||||
|
||||
|
||||
@app.get("/api/jobs/{job_id}/download_log")
|
||||
def get_job_log_file(job_id: str) -> FileResponse:
|
||||
"""Download the log file for a job."""
|
||||
@@ -750,6 +908,14 @@ def main() -> None:
|
||||
help=("Directory for SQLite database (jobs + settings persistence) "
|
||||
f"(default: {default_data_dir})"),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--max-concurrent-jobs",
|
||||
type=int,
|
||||
default=1,
|
||||
help=("How many inference jobs may run at once (default: 1). Each gets its own "
|
||||
"long-lived worker that keeps its model loaded between jobs, so raising "
|
||||
"this loads one copy of the model per worker."),
|
||||
)
|
||||
parser.add_argument(
|
||||
"--verbose",
|
||||
action="store_true",
|
||||
@@ -762,6 +928,7 @@ def main() -> None:
|
||||
data_dir = Path(args.data_dir).resolve()
|
||||
upload_dir = str(data_dir / "uploads")
|
||||
datasets_upload_dir = str(data_dir / "uploads" / "datasets")
|
||||
use_server_data_dir(data_dir)
|
||||
|
||||
create_local_env(args.host, args.port)
|
||||
|
||||
@@ -775,6 +942,8 @@ def main() -> None:
|
||||
log_dir=log_dir,
|
||||
verbose=args.verbose,
|
||||
database=database,
|
||||
max_concurrent_jobs=args.max_concurrent_jobs,
|
||||
upload_dir=upload_dir,
|
||||
)
|
||||
|
||||
logger.info("Output directory: %s", output_dir)
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,307 @@
|
||||
'use client';
|
||||
|
||||
import { AlertTriangle, Loader2, PenLine, RefreshCw } from 'lucide-react';
|
||||
import * as React from 'react';
|
||||
|
||||
import CreateJobModal from '@/components/jobs/CreateJobModal';
|
||||
import { EDITABLE_STATUSES, SceneBoard } from '@/components/scenes/SceneBoard';
|
||||
import { MergedSceneDialog } from '@/components/scenes/MergedSceneDialog';
|
||||
import { SceneQueueDialog } from '@/components/scenes/SceneQueueDialog';
|
||||
import { ReviseClipDialog } from '@/components/scenes/ReviseClipDialog';
|
||||
import { TrimClipDialog } from '@/components/scenes/TrimClipDialog';
|
||||
import { WriteSceneDialog } from '@/components/scenes/WriteSceneDialog';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card } from '@/components/ui/card';
|
||||
import { NativeSelect } from '@/components/ui/native-select';
|
||||
import { toast } from 'sonner';
|
||||
import { dequeueJob, getJobsList, mergeScene, queueJobs, updateJob, type MergedScene } from '@/lib/api';
|
||||
import { attachedLastFrames, attachLastFrame, deferredLastFrameSource, detachLastFrame } from '@/lib/continuation';
|
||||
import type { SceneQueuePlan } from '@/lib/sceneQueue';
|
||||
import { buildScenes, type Scene, type SceneClip } from '@/lib/scenes';
|
||||
import type { Job } from '@/lib/types';
|
||||
|
||||
const POLL_MS = 3000;
|
||||
|
||||
export default function ScenesPage() {
|
||||
const [jobs, setJobs] = React.useState<Job[]>([]);
|
||||
const [isLoading, setIsLoading] = React.useState(true);
|
||||
const [error, setError] = React.useState<string | null>(null);
|
||||
const [reloadKey, setReloadKey] = React.useState(0);
|
||||
const [selectedKey, setSelectedKey] = React.useState<string | null>(null);
|
||||
const [openJob, setOpenJob] = React.useState<Job | null>(null);
|
||||
// which re-run of a clip number the user picked, by clip
|
||||
const [choices, setChoices] = React.useState<Record<string, string>>({});
|
||||
const [writing, setWriting] = React.useState(false);
|
||||
const [queueTarget, setQueueTarget] = React.useState<Scene | null>(null);
|
||||
const [merged, setMerged] = React.useState<(MergedScene & { title: string }) | null>(null);
|
||||
const [trimTarget, setTrimTarget] = React.useState<Job | null>(null);
|
||||
const [reviseTarget, setReviseTarget] = React.useState<SceneClip | null>(null);
|
||||
// Bumped per job id whenever its video is edited in place, to cache-bust every
|
||||
// <video> for that job (the dialog's own preview and the scene board's). Owned
|
||||
// here, not by the dialog, so it survives the dialog re-rendering with a fresh
|
||||
// job object after each edit -- a per-dialog counter got reset by that same
|
||||
// re-render before the browser ever saw the new URL.
|
||||
const [videoVersions, setVideoVersions] = React.useState<Record<string, number>>({});
|
||||
const bumpVideoVersion = (jobId: string) =>
|
||||
setVideoVersions((v) => ({ ...v, [jobId]: (v[jobId] ?? 0) + 1 }));
|
||||
|
||||
React.useEffect(() => {
|
||||
let cancelled = false;
|
||||
async function load() {
|
||||
try {
|
||||
const list = await getJobsList('inference');
|
||||
if (cancelled) return;
|
||||
setJobs(list);
|
||||
setError(null);
|
||||
} catch (e) {
|
||||
if (!cancelled) setError(e instanceof Error ? e.message : 'Failed to load jobs');
|
||||
} finally {
|
||||
if (!cancelled) setIsLoading(false);
|
||||
}
|
||||
}
|
||||
void load();
|
||||
return () => {
|
||||
cancelled = true;
|
||||
};
|
||||
}, [reloadKey]);
|
||||
|
||||
function reload() {
|
||||
setIsLoading(true);
|
||||
setReloadKey((k) => k + 1);
|
||||
}
|
||||
|
||||
// While anything is queued or running, keep the board current without the
|
||||
// loading state (which would blank or flicker it).
|
||||
const hasActive = jobs.some((j) => j.status === 'queued' || j.status === 'running');
|
||||
React.useEffect(() => {
|
||||
if (!hasActive) return;
|
||||
const interval = setInterval(() => {
|
||||
getJobsList('inference')
|
||||
.then(setJobs)
|
||||
.catch(() => {});
|
||||
}, POLL_MS);
|
||||
return () => clearInterval(interval);
|
||||
}, [hasActive]);
|
||||
|
||||
// Attach the previous clip's last frame to `clip` and rewrite its prompt to open on
|
||||
// it. The frame is a reference the server resolves when the clip runs, so the
|
||||
// previous clip doesn't have to be finished yet. Returns the picture label.
|
||||
async function attachPreviousFrame(clip: SceneClip, previous: SceneClip): Promise<string> {
|
||||
const result = attachLastFrame(
|
||||
clip.job.prompt,
|
||||
clip.job.references ?? [],
|
||||
deferredLastFrameSource(previous.job.id),
|
||||
);
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
await updateJob(clip.job.id, { references: result.references, prompt: result.prompt });
|
||||
return result.label;
|
||||
}
|
||||
|
||||
async function useLastFrame(clip: SceneClip, previous: SceneClip) {
|
||||
try {
|
||||
const label = await attachPreviousFrame(clip, previous);
|
||||
toast.success(`Clip ${clip.position} now starts from the last frame of clip ${previous.position} (${label}).`);
|
||||
reload();
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't use the last frame.");
|
||||
}
|
||||
}
|
||||
|
||||
// Undo the above: take every attached last frame back off the clip.
|
||||
async function stopUsingLastFrame(clip: SceneClip) {
|
||||
try {
|
||||
let prompt = clip.job.prompt;
|
||||
let references = clip.job.references ?? [];
|
||||
// Newest first: only the most recently added image can be removed cleanly.
|
||||
for (const { source } of attachedLastFrames(references).reverse()) {
|
||||
const result = detachLastFrame(prompt, references, source);
|
||||
if (!result.ok) throw new Error(result.error);
|
||||
prompt = result.prompt;
|
||||
references = result.references;
|
||||
}
|
||||
await updateJob(clip.job.id, { references, prompt });
|
||||
toast.success(`Clip ${clip.position} no longer starts from a previous clip's last frame.`);
|
||||
reload();
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't remove the last frame.");
|
||||
}
|
||||
}
|
||||
|
||||
// Join the scene's clips, in the order shown (the take chosen for each), into one video.
|
||||
async function mergeSceneClips(scene: Scene) {
|
||||
try {
|
||||
const result = await mergeScene(scene.clips.map((c) => c.job.id), scene.title);
|
||||
setMerged({ ...result, title: scene.title });
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't merge the scene.");
|
||||
}
|
||||
}
|
||||
|
||||
async function queueClip(clip: SceneClip) {
|
||||
try {
|
||||
await queueJobs([clip.job.id]);
|
||||
toast.success(`Queued clip ${clip.position}.`);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't queue the clip.");
|
||||
}
|
||||
reload();
|
||||
}
|
||||
|
||||
async function dequeueClip(clip: SceneClip) {
|
||||
try {
|
||||
await dequeueJob(clip.job.id);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't remove the clip from the queue.");
|
||||
}
|
||||
reload();
|
||||
}
|
||||
|
||||
// Link clips to the frames before them (if asked), then queue them all in scene order.
|
||||
async function queueScene(plan: SceneQueuePlan) {
|
||||
try {
|
||||
for (const { clip, previous } of plan.chain) {
|
||||
await attachPreviousFrame(clip, previous).catch((e) => {
|
||||
throw new Error(`Clip ${clip.position}: ${e instanceof Error ? e.message : e}. Nothing was queued.`);
|
||||
});
|
||||
}
|
||||
await queueJobs(plan.queue.map((c) => c.job.id));
|
||||
const n = plan.queue.length;
|
||||
toast.success(`Queued ${n} clip${n === 1 ? '' : 's'}.`);
|
||||
setQueueTarget(null);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't queue the scene.");
|
||||
}
|
||||
reload();
|
||||
}
|
||||
|
||||
const scenes = React.useMemo(() => buildScenes(jobs, choices), [jobs, choices]);
|
||||
const scene = scenes.find((s) => s.key === selectedKey) ?? scenes[0];
|
||||
// The shown takes of the clips on either side of the one being revised.
|
||||
const reviseNeighbors = React.useMemo(() => {
|
||||
if (!reviseTarget) return [];
|
||||
for (const s of scenes) {
|
||||
const i = s.clips.findIndex((c) => c.job.id === reviseTarget.job.id);
|
||||
if (i >= 0) return [s.clips[i - 1], s.clips[i + 1]].filter((c) => c !== undefined).map((c) => c.job);
|
||||
}
|
||||
return [];
|
||||
}, [scenes, reviseTarget]);
|
||||
|
||||
return (
|
||||
<div className="mx-auto w-full max-w-[1200px] px-4 pb-12 pt-4">
|
||||
<Card className="p-6">
|
||||
<div className="mb-1 flex flex-wrap items-center gap-3">
|
||||
<h2 className="text-2xl font-semibold text-foreground">Scenes</h2>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="ml-auto gap-1.5"
|
||||
onClick={() => setWriting(true)}
|
||||
>
|
||||
<PenLine className="size-3.5" aria-hidden />
|
||||
Write scene
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="gap-1.5"
|
||||
onClick={reload}
|
||||
disabled={isLoading}
|
||||
>
|
||||
<RefreshCw className={isLoading ? 'size-3.5 animate-spin' : 'size-3.5'} aria-hidden />
|
||||
Refresh
|
||||
</Button>
|
||||
</div>
|
||||
<p className="mb-6 text-sm text-muted-foreground">
|
||||
Every shot across your jobs, in order. Jobs named with a shared stem and a number, like{' '}
|
||||
<code className="font-mono text-xs">wolf-lunch-clip-01</code>,{' '}
|
||||
<code className="font-mono text-xs">wolf-lunch-clip-02</code>, are grouped into one scene.
|
||||
</p>
|
||||
|
||||
{isLoading && jobs.length === 0 ? (
|
||||
<div className="flex items-center gap-3 p-8 text-muted-foreground">
|
||||
<Loader2 className="h-6 w-6 animate-spin text-primary" />
|
||||
<span>Loading scenes…</span>
|
||||
</div>
|
||||
) : error && jobs.length === 0 ? (
|
||||
<div role="alert" className="flex flex-col items-center gap-3 py-8 text-center">
|
||||
<AlertTriangle className="size-6 text-destructive" aria-hidden />
|
||||
<p className="max-w-md text-sm text-muted-foreground">{error}</p>
|
||||
<Button type="button" variant="outline" onClick={reload}>
|
||||
Try Again
|
||||
</Button>
|
||||
</div>
|
||||
) : !scene ? (
|
||||
<p className="py-8 text-center text-muted-foreground">
|
||||
No inference jobs yet. Create some and they'll show up here.
|
||||
</p>
|
||||
) : (
|
||||
<div className="flex flex-col gap-6">
|
||||
{scenes.length > 1 && (
|
||||
<label className="flex max-w-md flex-col gap-1 text-xs text-muted-foreground">
|
||||
Scene
|
||||
<NativeSelect value={scene.key} onChange={(e) => setSelectedKey(e.target.value)}>
|
||||
{scenes.map((s) => (
|
||||
<option key={s.key} value={s.key}>
|
||||
{s.title} ({s.clips.length} clip{s.clips.length === 1 ? '' : 's'})
|
||||
</option>
|
||||
))}
|
||||
</NativeSelect>
|
||||
</label>
|
||||
)}
|
||||
<SceneBoard
|
||||
scene={scene}
|
||||
onOpenJob={setOpenJob}
|
||||
onUseLastFrame={useLastFrame}
|
||||
onDetachLastFrame={stopUsingLastFrame}
|
||||
onQueueScene={setQueueTarget}
|
||||
onMergeScene={mergeSceneClips}
|
||||
onQueueClip={queueClip}
|
||||
onDequeueClip={dequeueClip}
|
||||
onTrimClip={(clip) => setTrimTarget(clip.job)}
|
||||
onReviseClip={setReviseTarget}
|
||||
onChooseTake={(takeKey, jobId) => setChoices((c) => ({ ...c, [takeKey]: jobId }))}
|
||||
videoVersionFor={(jobId) => videoVersions[jobId] ?? 0}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
</Card>
|
||||
|
||||
<MergedSceneDialog merged={merged} onClose={() => setMerged(null)} />
|
||||
<WriteSceneDialog open={writing} onClose={() => setWriting(false)} onCreated={reload} />
|
||||
<ReviseClipDialog
|
||||
clip={reviseTarget}
|
||||
neighbors={reviseNeighbors}
|
||||
onClose={() => setReviseTarget(null)}
|
||||
onCreated={reload}
|
||||
/>
|
||||
<SceneQueueDialog scene={queueTarget} onConfirm={queueScene} onClose={() => setQueueTarget(null)} />
|
||||
<TrimClipDialog
|
||||
clip={trimTarget}
|
||||
videoVersion={trimTarget ? (videoVersions[trimTarget.id] ?? 0) : 0}
|
||||
onChanged={(job) => {
|
||||
setTrimTarget(job);
|
||||
bumpVideoVersion(job.id);
|
||||
getJobsList('inference').then(setJobs).catch(() => {});
|
||||
}}
|
||||
onClose={() => setTrimTarget(null)}
|
||||
/>
|
||||
|
||||
{openJob && (
|
||||
<CreateJobModal
|
||||
isOpen
|
||||
readOnly={!EDITABLE_STATUSES.includes(openJob.status)}
|
||||
editingJob={openJob}
|
||||
jobType={(openJob.job_type ?? 'inference') as never}
|
||||
workloadType={openJob.workload_type ?? 't2v'}
|
||||
onClose={() => setOpenJob(null)}
|
||||
onSuccess={() => {
|
||||
setOpenJob(null);
|
||||
reload();
|
||||
}}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,93 @@
|
||||
import { render, screen, waitFor, within } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import CreateJobButton from './CreateJobButton';
|
||||
import { getDatasets, getModels } from '@/lib/api';
|
||||
|
||||
vi.mock('@/lib/api', () => ({
|
||||
createJob: vi.fn(),
|
||||
getModels: vi.fn(),
|
||||
getDatasets: vi.fn(),
|
||||
uploadImage: vi.fn(),
|
||||
uploadMedia: vi.fn(),
|
||||
updateJob: vi.fn(),
|
||||
getJobReferenceUrl: vi.fn(),
|
||||
getSettings: vi.fn(),
|
||||
updateSettings: vi.fn(),
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(getModels).mockResolvedValue([
|
||||
{ id: 'wan/t2v-1.3b', label: 'Wan T2V' },
|
||||
]);
|
||||
vi.mocked(getDatasets).mockResolvedValue([]);
|
||||
});
|
||||
|
||||
// The real CreateJobModal, unlike CreateJobButton.test.tsx's stub, so the
|
||||
// dialog's focus and pointer-events cleanup run. It opens a frame after the
|
||||
// menu selection (see CreateJobButton's openModal), so tests wait for it.
|
||||
describe('CreateJobButton with the real dialog', () => {
|
||||
it('opens the workload menu on click and selects an item', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateJobButton jobType="inference" />);
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Create Job' }));
|
||||
await user.click(screen.getByRole('menuitem', { name: /I2V/i }));
|
||||
|
||||
expect(
|
||||
await screen.findByRole('dialog', { name: 'New Inference Job (I2V)' }),
|
||||
).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('opens and operates the workload menu from the keyboard', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateJobButton jobType="inference" />);
|
||||
|
||||
const trigger = screen.getByRole('button', { name: 'Create Job' });
|
||||
trigger.focus();
|
||||
await user.keyboard('{Enter}');
|
||||
|
||||
const firstItem = await screen.findByRole('menuitem', { name: /T2V/i });
|
||||
expect(firstItem).toHaveFocus();
|
||||
await user.keyboard('{Enter}');
|
||||
|
||||
expect(
|
||||
await screen.findByRole('dialog', { name: 'New Inference Job (T2V)' }),
|
||||
).toBeInTheDocument();
|
||||
await user.keyboard('{Escape}');
|
||||
await waitFor(() =>
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument(),
|
||||
);
|
||||
await waitFor(() =>
|
||||
expect(document.body.style.pointerEvents).not.toBe('none'),
|
||||
);
|
||||
expect(trigger).toHaveFocus();
|
||||
});
|
||||
|
||||
it.each(['inference', 'finetuning', 'distillation'] as const)(
|
||||
'restores page interaction after closing the real %s dialog',
|
||||
async (jobType) => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateJobButton jobType={jobType} />);
|
||||
const trigger = screen.getByRole('button', { name: 'Create Job' });
|
||||
|
||||
// Keep the real Dialog mounted: mocking it hides conflicting Radix layers.
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
await user.click(trigger);
|
||||
await user.click(screen.getAllByRole('menuitem')[0]);
|
||||
const dialog = await screen.findByRole('dialog');
|
||||
await user.click(
|
||||
within(dialog).getByRole('button', { name: 'Close' }),
|
||||
);
|
||||
await waitFor(() =>
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument(),
|
||||
);
|
||||
await waitFor(() =>
|
||||
expect(document.body.style.pointerEvents).not.toBe('none'),
|
||||
);
|
||||
expect(trigger).toHaveFocus();
|
||||
}
|
||||
},
|
||||
);
|
||||
});
|
||||
@@ -1,26 +1,34 @@
|
||||
import { render, screen, waitFor, within } from '@testing-library/react';
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import { toast } from 'sonner';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import CreateJobButton from './CreateJobButton';
|
||||
import { getDatasets, getModels } from '@/lib/api';
|
||||
|
||||
vi.mock('@/lib/api', () => ({
|
||||
createJob: vi.fn(),
|
||||
getModels: vi.fn(),
|
||||
getDatasets: vi.fn(),
|
||||
uploadImage: vi.fn(),
|
||||
getSettings: vi.fn(),
|
||||
updateSettings: vi.fn(),
|
||||
vi.mock('sonner', () => ({ toast: { error: vi.fn(), warning: vi.fn() } }));
|
||||
|
||||
vi.mock('./CreateJobModal', () => ({
|
||||
default: ({
|
||||
isOpen,
|
||||
workloadType,
|
||||
prefillJob,
|
||||
}: {
|
||||
isOpen: boolean;
|
||||
workloadType: string;
|
||||
prefillJob?: { name?: string; prompt: string } | null;
|
||||
}) =>
|
||||
isOpen ? (
|
||||
<div
|
||||
role="dialog"
|
||||
data-workload-type={workloadType}
|
||||
data-prefill-name={prefillJob?.name ?? ''}
|
||||
data-prefill-prompt={prefillJob?.prompt ?? ''}
|
||||
>
|
||||
Create job form
|
||||
</div>
|
||||
) : null,
|
||||
}));
|
||||
|
||||
beforeEach(() => {
|
||||
vi.mocked(getModels).mockResolvedValue([
|
||||
{ id: 'wan/t2v-1.3b', label: 'Wan T2V' },
|
||||
]);
|
||||
vi.mocked(getDatasets).mockResolvedValue([]);
|
||||
});
|
||||
|
||||
describe('CreateJobButton', () => {
|
||||
it('opens the workload menu on click and selects an item', async () => {
|
||||
const user = userEvent.setup();
|
||||
@@ -29,9 +37,14 @@ describe('CreateJobButton', () => {
|
||||
await user.click(screen.getByRole('button', { name: 'Create Job' }));
|
||||
await user.click(screen.getByRole('menuitem', { name: /I2V/i }));
|
||||
|
||||
expect(
|
||||
screen.getByRole('dialog', { name: 'New Inference Job (I2V)' }),
|
||||
).toBeInTheDocument();
|
||||
// The dialog opens a frame after the menu selection (see CreateJobButton's
|
||||
// openModal: deferred via requestAnimationFrame to dodge a Radix
|
||||
// DropdownMenu + Dialog pointer-events race), so wait for it rather than
|
||||
// asserting synchronously.
|
||||
expect(await screen.findByRole('dialog')).toHaveAttribute(
|
||||
'data-workload-type',
|
||||
'i2v',
|
||||
);
|
||||
});
|
||||
|
||||
it('opens and operates the workload menu from the keyboard', async () => {
|
||||
@@ -46,42 +59,75 @@ describe('CreateJobButton', () => {
|
||||
expect(firstItem).toHaveFocus();
|
||||
await user.keyboard('{Enter}');
|
||||
|
||||
expect(
|
||||
screen.getByRole('dialog', { name: 'New Inference Job (T2V)' }),
|
||||
).toBeInTheDocument();
|
||||
await user.keyboard('{Escape}');
|
||||
await waitFor(() =>
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument(),
|
||||
expect(await screen.findByRole('dialog')).toHaveAttribute(
|
||||
'data-workload-type',
|
||||
't2v',
|
||||
);
|
||||
await waitFor(() =>
|
||||
expect(document.body.style.pointerEvents).not.toBe('none'),
|
||||
);
|
||||
expect(trigger).toHaveFocus();
|
||||
});
|
||||
|
||||
it.each(['inference', 'finetuning', 'distillation'] as const)(
|
||||
'restores page interaction after closing the real %s dialog',
|
||||
async (jobType) => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateJobButton jobType={jobType} />);
|
||||
const trigger = screen.getByRole('button', { name: 'Create Job' });
|
||||
describe('importing a job file', () => {
|
||||
const file = (contents: string, name = 'job.json') =>
|
||||
new File([contents], name, { type: 'application/json' });
|
||||
|
||||
// Keep the real Dialog mounted: mocking it hides conflicting Radix layers.
|
||||
for (let attempt = 0; attempt < 2; attempt++) {
|
||||
await user.click(trigger);
|
||||
await user.click(screen.getAllByRole('menuitem')[0]);
|
||||
const dialog = screen.getByRole('dialog');
|
||||
await user.click(
|
||||
within(dialog).getByRole('button', { name: 'Close' }),
|
||||
);
|
||||
await waitFor(() =>
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument(),
|
||||
);
|
||||
await waitFor(() =>
|
||||
expect(document.body.style.pointerEvents).not.toBe('none'),
|
||||
);
|
||||
expect(trigger).toHaveFocus();
|
||||
}
|
||||
},
|
||||
);
|
||||
it('opens the form for the file\'s workload with the job pre-filled', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateJobButton jobType="inference" />);
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Create Job' }));
|
||||
await user.click(screen.getByRole('menuitem', { name: /Import job JSON/ }));
|
||||
await user.upload(
|
||||
screen.getByLabelText('Import job JSON file'),
|
||||
file(
|
||||
JSON.stringify({
|
||||
model_id: 'MiniMaxAI/MiniMax-H3',
|
||||
name: 'from-file',
|
||||
workload_type: 'i2v',
|
||||
prompt: ['summary:', 'Hi.'],
|
||||
}),
|
||||
),
|
||||
);
|
||||
|
||||
const dialog = await screen.findByRole('dialog');
|
||||
expect(dialog).toHaveAttribute('data-workload-type', 'i2v');
|
||||
expect(dialog).toHaveAttribute('data-prefill-name', 'from-file');
|
||||
expect(dialog).toHaveAttribute('data-prefill-prompt', 'summary:\nHi.');
|
||||
});
|
||||
|
||||
it('reports an unreadable file instead of opening the form', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateJobButton jobType="inference" />);
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Create Job' }));
|
||||
await user.click(screen.getByRole('menuitem', { name: /Import job JSON/ }));
|
||||
await user.upload(screen.getByLabelText('Import job JSON file'), file('not json', 'bad.json'));
|
||||
|
||||
await vi.waitFor(() => expect(toast.error).toHaveBeenCalled());
|
||||
expect(vi.mocked(toast.error).mock.calls.at(-1)?.[0]).toContain("bad.json");
|
||||
expect(screen.queryByRole('dialog')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('a normal open after an import starts clean', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateJobButton jobType="inference" />);
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Create Job' }));
|
||||
await user.click(screen.getByRole('menuitem', { name: /Import job JSON/ }));
|
||||
await user.upload(
|
||||
screen.getByLabelText('Import job JSON file'),
|
||||
file(JSON.stringify({ model_id: 'm', name: 'from-file', prompt: 'p' })),
|
||||
);
|
||||
expect(await screen.findByRole('dialog')).toHaveAttribute('data-prefill-name', 'from-file');
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Create Job' }));
|
||||
await user.click(await screen.findByRole('menuitem', { name: /T2V/i }));
|
||||
await vi.waitFor(() => expect(screen.getByRole('dialog')).toHaveAttribute('data-prefill-name', ''));
|
||||
});
|
||||
|
||||
it('is not offered on the training pages', async () => {
|
||||
const user = userEvent.setup();
|
||||
render(<CreateJobButton jobType="finetuning" />);
|
||||
await user.click(screen.getByRole('button', { name: 'Create Job' }));
|
||||
expect(screen.queryByRole('menuitem', { name: /Import job JSON/ })).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -6,7 +6,10 @@ import { DropdownMenu } from 'radix-ui';
|
||||
|
||||
import CreateJobModal from '@/components/jobs/CreateJobModal';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { toast } from 'sonner';
|
||||
import { WORKLOAD_OPTIONS } from '@/lib/jobConfig';
|
||||
import { parseJobJson } from '@/lib/jobJson';
|
||||
import type { JobLike } from '@/lib/jobToFields';
|
||||
import type { JobType } from '@/lib/types';
|
||||
import { triggerRefresh } from '@/stores/jobsRefresh';
|
||||
|
||||
@@ -23,9 +26,32 @@ export default function CreateJobButton({ jobType }: CreateJobButtonProps) {
|
||||
options[0]?.type ?? 't2v',
|
||||
);
|
||||
|
||||
function openModal(type: string) {
|
||||
// An imported job file pre-fills the form; a normal open starts clean.
|
||||
const [prefillJob, setPrefillJob] = React.useState<JobLike | null>(null);
|
||||
const fileInputRef = React.useRef<HTMLInputElement>(null);
|
||||
|
||||
async function handleImportFile(e: React.ChangeEvent<HTMLInputElement>) {
|
||||
const file = e.target.files?.[0];
|
||||
e.target.value = ''; // allow re-picking the same file
|
||||
if (!file) return;
|
||||
const result = parseJobJson(await file.text());
|
||||
if (!result.ok) {
|
||||
toast.error(`Couldn't import ${file.name}: ${result.error}`);
|
||||
return;
|
||||
}
|
||||
openModal(result.workloadType, result.job);
|
||||
}
|
||||
|
||||
function openModal(type: string, prefill: JobLike | null = null) {
|
||||
setWorkloadType(type);
|
||||
setModalOpen(true);
|
||||
setPrefillJob(prefill);
|
||||
// Opening the Dialog synchronously inside the DropdownMenu.Item's
|
||||
// onSelect races the two Radix layers' pointer-events body lock/unlock:
|
||||
// the dropdown's own close cleanup can lose that race, leaving <body>
|
||||
// stuck with `pointer-events: none` after this dialog later closes --
|
||||
// silently blocking every click, including this same button. Deferring
|
||||
// to the next frame lets the dropdown finish closing first.
|
||||
requestAnimationFrame(() => setModalOpen(true));
|
||||
}
|
||||
|
||||
function handleSuccess() {
|
||||
@@ -61,9 +87,32 @@ export default function CreateJobButton({ jobType }: CreateJobButtonProps) {
|
||||
</span>
|
||||
</DropdownMenu.Item>
|
||||
))}
|
||||
{jobType === 'inference' && (
|
||||
<>
|
||||
<DropdownMenu.Separator className="my-1 h-px bg-border" />
|
||||
<DropdownMenu.Item
|
||||
onSelect={() => fileInputRef.current?.click()}
|
||||
className="flex min-h-11 cursor-pointer select-none flex-col justify-center px-4 py-2 text-left text-sm font-medium outline-none data-[highlighted]:bg-secondary"
|
||||
>
|
||||
Import job JSON…
|
||||
<span className="mt-0.5 block text-xs font-normal text-muted-foreground">
|
||||
Fill the form from a job file
|
||||
</span>
|
||||
</DropdownMenu.Item>
|
||||
</>
|
||||
)}
|
||||
</DropdownMenu.Content>
|
||||
</DropdownMenu.Portal>
|
||||
</DropdownMenu.Root>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".json,application/json"
|
||||
aria-label="Import job JSON file"
|
||||
tabIndex={-1}
|
||||
className="sr-only"
|
||||
onChange={handleImportFile}
|
||||
/>
|
||||
<CreateJobModal
|
||||
isOpen={modalOpen}
|
||||
onClose={() => setModalOpen(false)}
|
||||
@@ -75,6 +124,7 @@ export default function CreateJobButton({ jobType }: CreateJobButtonProps) {
|
||||
onSuccess={handleSuccess}
|
||||
jobType={jobType}
|
||||
workloadType={workloadType}
|
||||
prefillJob={prefillJob}
|
||||
/>
|
||||
</>
|
||||
);
|
||||
|
||||
@@ -4,15 +4,21 @@ import userEvent from '@testing-library/user-event';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import CreateJobModal from './CreateJobModal';
|
||||
import { createJob, getDatasets, getModels, uploadImage } from '@/lib/api';
|
||||
import { toast } from 'sonner';
|
||||
import { createJob, getDatasets, getModels, uploadImage, uploadMedia } from '@/lib/api';
|
||||
import { defaultOptionsStore } from '@/stores/defaultOptions';
|
||||
import { DEFAULT_OPTIONS } from '@/lib/defaultOptions';
|
||||
|
||||
vi.mock('sonner', () => ({ toast: { warning: vi.fn(), error: vi.fn(), success: vi.fn() } }));
|
||||
|
||||
vi.mock('@/lib/api', () => ({
|
||||
createJob: vi.fn(),
|
||||
getModels: vi.fn(),
|
||||
getDatasets: vi.fn(),
|
||||
uploadImage: vi.fn(),
|
||||
uploadMedia: vi.fn(),
|
||||
updateJob: vi.fn(),
|
||||
getJobReferenceUrl: (jobId: string, source: string) => `/api/jobs/${jobId}/reference?source=${encodeURIComponent(source)}`,
|
||||
getSettings: vi.fn(),
|
||||
updateSettings: vi.fn(),
|
||||
}));
|
||||
@@ -206,4 +212,219 @@ describe('CreateJobModal', () => {
|
||||
|
||||
await waitFor(() => expect(onSuccess).toHaveBeenCalledTimes(1));
|
||||
});
|
||||
|
||||
it('shows the H3 speech-tag console only for an H3 model and inserts at the prompt cursor', async () => {
|
||||
vi.mocked(getModels).mockResolvedValue([
|
||||
...MODELS,
|
||||
{ id: 'minimax-h3/i2v', label: 'MiniMax H3', type: 'i2v' },
|
||||
]);
|
||||
const user = userEvent.setup();
|
||||
renderModal({ workloadType: 'i2v' });
|
||||
|
||||
await screen.findByRole('option', { name: 'Wan T2V (wan/t2v-1.3b)' });
|
||||
expect(screen.queryByText(/Speech tags/)).not.toBeInTheDocument();
|
||||
|
||||
await user.selectOptions(
|
||||
screen.getByLabelText('Model'),
|
||||
'minimax-h3/i2v',
|
||||
);
|
||||
expect(await screen.findByText(/Speech tags/)).toBeInTheDocument();
|
||||
|
||||
const promptField = screen.getByLabelText('Prompt') as HTMLTextAreaElement;
|
||||
await user.type(promptField, 'Hello there.');
|
||||
promptField.setSelectionRange(5, 5);
|
||||
|
||||
await user.click(screen.getByRole('button', { name: '<pause>' }));
|
||||
|
||||
expect(promptField.value).toBe('Hello <pause> there.');
|
||||
// Focus and cursor return to the prompt field so tags can be chained.
|
||||
await waitFor(() => expect(promptField).toHaveFocus());
|
||||
});
|
||||
});
|
||||
|
||||
const H3 = { id: 'minimax-h3/i2v', label: 'MiniMax H3', type: 'i2v' };
|
||||
|
||||
const BASE_PROMPT = [
|
||||
'For the target video, <Picture 1> is fully referenced.',
|
||||
'',
|
||||
'integrated_multimodal_description: [Shot 1] A close-up. He (S1) says: <d>[English] Sorry.</d> <cutoff>',
|
||||
'',
|
||||
'overall_soundscape: Room tone.',
|
||||
'',
|
||||
'non_diegetic_music: N/A',
|
||||
].join('\n');
|
||||
|
||||
describe('CreateJobModal with a prefilled job', () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(getModels).mockResolvedValue([...MODELS, H3]);
|
||||
vi.mocked(toast.warning).mockClear();
|
||||
});
|
||||
|
||||
it('fills the form from the job and creates a new job on submit', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onSuccess } = renderModal({
|
||||
workloadType: 'i2v',
|
||||
prefillJob: {
|
||||
id: 'imported-1',
|
||||
model_id: H3.id,
|
||||
name: 'from-file',
|
||||
prompt: BASE_PROMPT,
|
||||
workload_type: 'i2v',
|
||||
references: [{ source: '/data/inputs/face.png', media_type: 'image' }],
|
||||
num_frames: 243,
|
||||
seed: 7,
|
||||
},
|
||||
});
|
||||
|
||||
await screen.findByRole('option', { name: /MiniMax H3/ });
|
||||
await waitFor(() => expect(screen.getByLabelText('Model')).toHaveValue(H3.id));
|
||||
expect(screen.getByLabelText('Name (optional)')).toHaveValue('from-file');
|
||||
expect(screen.getByLabelText('Prompt')).toHaveValue(BASE_PROMPT);
|
||||
expect(screen.getByText('face.png')).toBeInTheDocument();
|
||||
expect(screen.getByText('New Inference Job (I2V)')).toBeInTheDocument();
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Create Job' }));
|
||||
await waitFor(() => expect(onSuccess).toHaveBeenCalledTimes(1));
|
||||
const payload = vi.mocked(createJob).mock.calls[0][0] as unknown as Record<string, unknown>;
|
||||
expect(payload).toMatchObject({
|
||||
model_id: H3.id,
|
||||
name: 'from-file',
|
||||
prompt: BASE_PROMPT,
|
||||
num_frames: 243,
|
||||
seed: 7,
|
||||
references: [{ source: '/data/inputs/face.png', media_type: 'image' }],
|
||||
});
|
||||
});
|
||||
|
||||
it("warns when the file's model isn't offered for the workload", async () => {
|
||||
renderModal({
|
||||
workloadType: 't2v',
|
||||
prefillJob: { id: 'imported-2', model_id: 'not/a-real-model', prompt: 'p' },
|
||||
});
|
||||
await waitFor(() => expect(toast.warning).toHaveBeenCalled());
|
||||
expect(vi.mocked(toast.warning).mock.calls[0][0]).toContain('not/a-real-model');
|
||||
expect(screen.getByLabelText('Model')).toHaveValue(MODELS[0].id);
|
||||
});
|
||||
});
|
||||
|
||||
describe('CreateJobModal shot list on a raw prompt', () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(getModels).mockResolvedValue([...MODELS, H3]);
|
||||
});
|
||||
|
||||
async function openShotList(user: ReturnType<typeof userEvent.setup>) {
|
||||
await user.click(screen.getByRole('button', { name: 'Open shot list' }));
|
||||
return screen.findByRole('dialog', { name: 'Shot list' });
|
||||
}
|
||||
|
||||
it('edits dialogue inside integrated_multimodal_description and leaves the rest of the prompt alone', async () => {
|
||||
const user = userEvent.setup();
|
||||
renderModal({
|
||||
workloadType: 'i2v',
|
||||
prefillJob: {
|
||||
id: 'imported-3',
|
||||
model_id: H3.id,
|
||||
prompt: BASE_PROMPT,
|
||||
references: [{ source: '/data/face.png', media_type: 'image' }],
|
||||
},
|
||||
});
|
||||
await waitFor(() => expect(screen.getByLabelText('Model')).toHaveValue(H3.id));
|
||||
|
||||
const dialog = await openShotList(user);
|
||||
expect(within(dialog).getAllByText(/integrated_multimodal_description/).length).toBeGreaterThan(0);
|
||||
expect(within(dialog).queryByText('Retention analysis')).not.toBeInTheDocument();
|
||||
|
||||
const line = within(dialog).getByPlaceholderText('What the speaker says') as HTMLTextAreaElement;
|
||||
expect(line.value).toBe('Sorry.');
|
||||
await user.type(line, ' Really.');
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Apply' }));
|
||||
|
||||
await waitFor(() =>
|
||||
expect(screen.getByLabelText('Prompt')).toHaveValue(BASE_PROMPT.replace('Sorry.', 'Sorry. Really.')),
|
||||
);
|
||||
});
|
||||
|
||||
it('treats a plain prompt as the shot text and writes shot markers back over it', async () => {
|
||||
const user = userEvent.setup();
|
||||
renderModal({ workloadType: 't2v' });
|
||||
await screen.findByRole('option', { name: /MiniMax H3/ });
|
||||
await user.selectOptions(screen.getByLabelText('Model'), H3.id);
|
||||
await user.type(screen.getByLabelText('Prompt'), 'A dock at dawn.');
|
||||
|
||||
const dialog = await openShotList(user);
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Apply' }));
|
||||
await waitFor(() => expect(screen.getByLabelText('Prompt')).toHaveValue('[Shot 1] A dock at dawn.'));
|
||||
});
|
||||
|
||||
it('keeps retention analysis in step when the raw prompt has a detailed_description and retention_analysis', async () => {
|
||||
const user = userEvent.setup();
|
||||
const sections = [
|
||||
'subject_definitions:',
|
||||
'<Subject 1> is the courier.',
|
||||
'',
|
||||
'retention_analysis:',
|
||||
'<Subject 1> (appears in [Shot 1]): fully_preserved - jacket.',
|
||||
'',
|
||||
'detailed_description:',
|
||||
'[Shot 1] Wide shot. A rooftop with <Subject 1>.',
|
||||
].join('\n');
|
||||
renderModal({ workloadType: 't2v', prefillJob: { id: 'imported-4', model_id: H3.id, prompt: sections } });
|
||||
await waitFor(() => expect(screen.getByLabelText('Model')).toHaveValue(H3.id));
|
||||
|
||||
const dialog = await openShotList(user);
|
||||
expect(within(dialog).getByText('Retention analysis')).toBeInTheDocument();
|
||||
await user.click(within(dialog).getAllByRole('button', { name: /Add shot/ })[0]);
|
||||
await user.click(within(dialog).getAllByRole('button', { name: '<Subject 1>' })[1]);
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Apply' }));
|
||||
|
||||
const out = (screen.getByLabelText('Prompt') as HTMLTextAreaElement).value;
|
||||
expect(out).toContain('<Subject 1> (appears in [Shot 1], [Shot 2]): fully_preserved - jacket.');
|
||||
expect(out).toContain('subject_definitions:\n<Subject 1> is the courier.');
|
||||
});
|
||||
|
||||
it('is not offered for a non-H3 model', async () => {
|
||||
renderModal({ workloadType: 't2v' });
|
||||
await screen.findByRole('option', { name: 'Wan T2V (wan/t2v-1.3b)' });
|
||||
expect(screen.queryByRole('button', { name: 'Open shot list' })).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe('CreateJobModal reference previews', () => {
|
||||
beforeEach(() => {
|
||||
vi.mocked(getModels).mockResolvedValue([...MODELS, H3]);
|
||||
});
|
||||
|
||||
const job = {
|
||||
id: 'job-9',
|
||||
model_id: H3.id,
|
||||
name: 'diner-clip-01',
|
||||
prompt: BASE_PROMPT,
|
||||
workload_type: 'i2v',
|
||||
status: 'pending',
|
||||
references: [
|
||||
{ source: '/refs/dana.jpg', media_type: 'image' },
|
||||
{ source: '/refs/voice.mp3', media_type: 'audio' },
|
||||
],
|
||||
};
|
||||
|
||||
it("shows a saved job's image references from the server and an icon for other media", async () => {
|
||||
renderModal({ workloadType: 'i2v', editingJob: job as never });
|
||||
|
||||
const thumb = await screen.findByRole('img', { name: 'dana.jpg' });
|
||||
expect(thumb).toHaveAttribute('src', '/api/jobs/job-9/reference?source=%2Frefs%2Fdana.jpg');
|
||||
expect(screen.queryByRole('img', { name: 'voice.mp3' })).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('previews a newly uploaded image from the local file', async () => {
|
||||
const user = userEvent.setup();
|
||||
URL.createObjectURL = vi.fn(() => 'blob:local-preview');
|
||||
URL.revokeObjectURL = vi.fn();
|
||||
vi.mocked(uploadMedia).mockResolvedValue({ path: '/uploads/u1/marcus.jpg', media_type: 'image' });
|
||||
renderModal({ workloadType: 'i2v', editingJob: job as never });
|
||||
|
||||
await screen.findByRole('img', { name: 'dana.jpg' });
|
||||
await user.upload(screen.getByLabelText('References (Ref2VA)'), new File(['x'], 'marcus.jpg', { type: 'image/jpeg' }));
|
||||
|
||||
expect(await screen.findByRole('img', { name: 'marcus.jpg' })).toHaveAttribute('src', 'blob:local-preview');
|
||||
});
|
||||
});
|
||||
|
||||
@@ -24,6 +24,7 @@ import {
|
||||
createJob,
|
||||
updateJob,
|
||||
getDatasets,
|
||||
getJobReferenceUrl,
|
||||
getModels,
|
||||
uploadImage,
|
||||
uploadMedia,
|
||||
@@ -49,8 +50,19 @@ import {
|
||||
parseH3Prompt,
|
||||
serializeH3Prompt,
|
||||
type H3PromptFields,
|
||||
type H3PromptSection,
|
||||
} from '@/lib/h3Prompt';
|
||||
import { applyH3Tag, type H3Tag } from '@/lib/h3Tags';
|
||||
import { H3TagConsole } from '@/components/jobs/H3TagConsole';
|
||||
import { H3DialogueEditor } from '@/components/jobs/H3DialogueEditor';
|
||||
import { H3ShotListEditor } from '@/components/jobs/H3ShotListEditor';
|
||||
import { ReferenceThumbnail } from '@/components/jobs/ReferenceThumbnail';
|
||||
import { jobToFormFields, type JobLike } from '@/lib/jobToFields';
|
||||
import { readField, shotTarget, writeField, writeShotTarget } from '@/lib/h3PromptFields';
|
||||
import { toast } from 'sonner';
|
||||
|
||||
/** Prompt fields the speech-tag console can insert into. */
|
||||
type TagTarget = 'prompt' | H3PromptSection;
|
||||
|
||||
export interface CreateJobModalProps {
|
||||
isOpen: boolean;
|
||||
@@ -63,6 +75,8 @@ export interface CreateJobModalProps {
|
||||
workloadType: string;
|
||||
/** When set, the modal edits this pending job instead of creating a new one. */
|
||||
editingJob?: JobLike | null;
|
||||
/** Seed the form from this job (an imported job file) but still create a new job on submit. */
|
||||
prefillJob?: JobLike | null;
|
||||
/** Show the configuration without allowing changes (started/finished jobs). */
|
||||
readOnly?: boolean;
|
||||
}
|
||||
@@ -75,6 +89,7 @@ export default function CreateJobModal({
|
||||
jobType,
|
||||
workloadType,
|
||||
editingJob,
|
||||
prefillJob,
|
||||
readOnly = false,
|
||||
}: CreateJobModalProps) {
|
||||
const { options } = useStore(defaultOptionsStore);
|
||||
@@ -91,6 +106,16 @@ export default function CreateJobModal({
|
||||
const [lastImagePath, setLastImagePath] = React.useState('');
|
||||
const [references, setReferences] = React.useState<H3Reference[]>([]);
|
||||
const [isUploadingReference, setIsUploadingReference] = React.useState(false);
|
||||
// Local previews of references uploaded in this session, by reference id; a saved job's own
|
||||
// references are previewed from the server instead.
|
||||
const [uploadPreviews, setUploadPreviews] = React.useState<Record<string, string>>({});
|
||||
React.useEffect(() => {
|
||||
if (isOpen) return;
|
||||
setUploadPreviews((previews) => {
|
||||
Object.values(previews).forEach((url) => URL.revokeObjectURL(url));
|
||||
return {};
|
||||
});
|
||||
}, [isOpen]);
|
||||
const [referenceError, setReferenceError] = React.useState<string | null>(null);
|
||||
const [promptFields, setPromptFields] = React.useState<H3PromptFields>(
|
||||
EMPTY_H3_PROMPT_FIELDS,
|
||||
@@ -125,11 +150,85 @@ export default function CreateJobModal({
|
||||
// H3 is the only registered model with an end frame or references.
|
||||
const supportsLastImage = modelId.toLowerCase().includes('minimax-h3');
|
||||
const usingReferences = supportsLastImage && references.length > 0;
|
||||
// Same check, named for where it gates H3-only prompt features (speech tags)
|
||||
// rather than the end-frame/reference upload UI.
|
||||
const isH3Model = supportsLastImage;
|
||||
|
||||
// Which prompt field the speech-tag console inserts into: whichever one is
|
||||
// actually visible, tracked via each field's onFocus below.
|
||||
const [activeTagField, setActiveTagField] = React.useState<TagTarget>('prompt');
|
||||
const tagFieldRefs = React.useRef<Partial<Record<TagTarget, HTMLTextAreaElement | null>>>({});
|
||||
const showGuidedFields = usingReferences && useGuidedPrompt;
|
||||
React.useEffect(() => {
|
||||
setActiveTagField(showGuidedFields ? 'detailed_description' : 'prompt');
|
||||
}, [showGuidedFields]);
|
||||
|
||||
function getTagFieldValue(field: TagTarget): string {
|
||||
return field === 'prompt' ? prompt : promptFields[field];
|
||||
}
|
||||
function setTagFieldValue(field: TagTarget, value: string) {
|
||||
if (field === 'prompt') setPrompt(value);
|
||||
else setPromptField(field, value);
|
||||
}
|
||||
|
||||
function insertH3Tag(tag: H3Tag) {
|
||||
const field = activeTagField;
|
||||
const el = tagFieldRefs.current[field] ?? null;
|
||||
const currentValue = getTagFieldValue(field);
|
||||
const start = el?.selectionStart ?? currentValue.length;
|
||||
const end = el?.selectionEnd ?? currentValue.length;
|
||||
const result = applyH3Tag(currentValue, start, end, tag);
|
||||
setTagFieldValue(field, result.value);
|
||||
// The textarea re-renders with the new value before this runs; restore
|
||||
// focus and the cursor position so tags can be chained without reaching
|
||||
// for the mouse each time.
|
||||
requestAnimationFrame(() => {
|
||||
const node = tagFieldRefs.current[field];
|
||||
if (node) {
|
||||
node.focus();
|
||||
node.setSelectionRange(result.start, result.end);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Full-screen dialogue editor: opens with a copy of whichever field the
|
||||
// speech-tag console would currently insert into, and on Apply overwrites
|
||||
// that same field wholesale (not just an insertion at the cursor).
|
||||
const [dialogueEditorOpen, setDialogueEditorOpen] = React.useState(false);
|
||||
const dialogueEditorField = activeTagField;
|
||||
|
||||
// Shot list: writes into detailed_description AND retention_analysis
|
||||
// together, so unlike the speech-tag console/dialogue editor it isn't
|
||||
// scoped to "whichever field is focused" -- it's specific to those two.
|
||||
const [shotListOpen, setShotListOpen] = React.useState(false);
|
||||
// 'sections' edits the guided Detailed description; 'prompt' edits the shots
|
||||
// inside the raw prompt (base format, or a plain prompt).
|
||||
const [shotListMode, setShotListMode] = React.useState<'sections' | 'prompt'>('sections');
|
||||
function applyShotList(detailedDescription: string, retentionAnalysis: string) {
|
||||
setPromptFields((prev) => ({
|
||||
...prev,
|
||||
detailed_description: detailedDescription,
|
||||
retention_analysis: retentionAnalysis,
|
||||
}));
|
||||
}
|
||||
|
||||
const promptShots = React.useMemo(() => shotTarget(prompt), [prompt]);
|
||||
const promptSubjects = React.useMemo(() => readField(prompt, 'subject_definitions') ?? '', [prompt]);
|
||||
const promptRetention = React.useMemo(() => readField(prompt, 'retention_analysis'), [prompt]);
|
||||
const promptSyncsRetention = promptShots.field === 'detailed_description' && promptRetention !== null;
|
||||
|
||||
function applyShotListToPrompt(shotText: string, retentionAnalysis: string) {
|
||||
let next = writeShotTarget(prompt, promptShots, shotText);
|
||||
if (promptSyncsRetention) next = writeField(next, 'retention_analysis', retentionAnalysis);
|
||||
setPrompt(next);
|
||||
}
|
||||
|
||||
// JobCard re-renders on every job-list poll, so `editingJob` is a fresh
|
||||
// object each time. Effects must depend on these, never on the object.
|
||||
const editingJobId = editingJob?.id ?? null;
|
||||
const editingJobModelId = editingJob?.model_id ?? null;
|
||||
const seedJob = editingJob ?? prefillJob ?? null;
|
||||
const seedJobId = seedJob?.id ?? null;
|
||||
const seedJobModelId = seedJob?.model_id ?? null;
|
||||
const isPrefill = !editingJob && !!prefillJob;
|
||||
|
||||
// Layerwise offload and FSDP compete for the DiT weights and FastVideoArgs
|
||||
// silently picks a winner (fastvideo_args.py:859); resolve it visibly here.
|
||||
@@ -208,10 +307,10 @@ export default function CreateJobModal({
|
||||
const justOpened = isOpen && !justOpenedRef.current;
|
||||
justOpenedRef.current = isOpen;
|
||||
if (!justOpened) return;
|
||||
if (editingJob) {
|
||||
if (seedJob) {
|
||||
// Must not fall through to the defaults below: a partially-seeded
|
||||
// form silently edits values the user never saw.
|
||||
const f = jobToFormFields(editingJob);
|
||||
const f = jobToFormFields(seedJob);
|
||||
setModelId(f.modelId);
|
||||
setName(f.name);
|
||||
setPrompt(f.prompt);
|
||||
@@ -300,7 +399,7 @@ export default function CreateJobModal({
|
||||
setRealScoreModelPath('');
|
||||
setFakeScoreModelPath('');
|
||||
}
|
||||
}, [isOpen, workloadType, inferenceWorkload, options, editingJobId]);
|
||||
}, [isOpen, workloadType, inferenceWorkload, options, seedJobId]);
|
||||
|
||||
// Load the models available for this workload.
|
||||
React.useEffect(() => {
|
||||
@@ -323,7 +422,7 @@ export default function CreateJobModal({
|
||||
// When editing, the job's own model wins over the workload default --
|
||||
// this resolves after the seeding effect, so choosing a default here
|
||||
// would silently swap the model out from under the user.
|
||||
const editedId = editingJobModelId;
|
||||
const editedId = seedJobModelId;
|
||||
const chosen =
|
||||
editedId && ids.includes(editedId)
|
||||
? editedId
|
||||
@@ -331,6 +430,11 @@ export default function CreateJobModal({
|
||||
? defaultId
|
||||
: (list[0]?.id ?? '');
|
||||
setModelId(chosen);
|
||||
if (isPrefill && editedId && chosen !== editedId) {
|
||||
toast.warning(
|
||||
`${editedId} isn't available for ${inferenceWorkload.toUpperCase()}${chosen ? `, so ${chosen} is selected instead` : ''}.`,
|
||||
);
|
||||
}
|
||||
if (workloadType === 'dmd_t2v') {
|
||||
setRealScoreModelPath(chosen);
|
||||
setFakeScoreModelPath(chosen);
|
||||
@@ -351,7 +455,7 @@ export default function CreateJobModal({
|
||||
return () => {
|
||||
stale = true;
|
||||
};
|
||||
}, [isOpen, inferenceWorkload, workloadType, editingJobModelId]);
|
||||
}, [isOpen, inferenceWorkload, workloadType, seedJobModelId, isPrefill]);
|
||||
|
||||
// Training jobs need a dataset; load the ready datasets when relevant.
|
||||
React.useEffect(() => {
|
||||
@@ -443,10 +547,15 @@ export default function CreateJobModal({
|
||||
setReferenceError(null);
|
||||
try {
|
||||
const { path, media_type } = await uploadMedia(file);
|
||||
const id = `${Date.now()}-${file.name}`;
|
||||
if (media_type === 'image') {
|
||||
const url = URL.createObjectURL(file);
|
||||
setUploadPreviews((previews) => ({ ...previews, [id]: url }));
|
||||
}
|
||||
const next: H3Reference[] = [
|
||||
...references,
|
||||
{
|
||||
id: `${Date.now()}-${file.name}`,
|
||||
id,
|
||||
source: path,
|
||||
media_type,
|
||||
fileName: file.name,
|
||||
@@ -824,6 +933,14 @@ export default function CreateJobModal({
|
||||
key={reference.id}
|
||||
className="flex items-center gap-2 text-xs text-muted-foreground"
|
||||
>
|
||||
<ReferenceThumbnail
|
||||
mediaType={reference.media_type}
|
||||
src={
|
||||
uploadPreviews[reference.id] ??
|
||||
(editingJob ? getJobReferenceUrl(editingJob.id, reference.source) : null)
|
||||
}
|
||||
alt={reference.fileName}
|
||||
/>
|
||||
<code className="font-mono text-accent-blue">
|
||||
{labelReferences(references)[index]}
|
||||
</code>
|
||||
@@ -862,6 +979,22 @@ export default function CreateJobModal({
|
||||
</FieldRow>
|
||||
)}
|
||||
|
||||
{isH3Model && isInference && workloadType !== 't2i' && (
|
||||
<div className="flex flex-col gap-2">
|
||||
<H3TagConsole onInsert={insertH3Tag} disabled={isSubmitting} />
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="self-start"
|
||||
onClick={() => setDialogueEditorOpen(true)}
|
||||
disabled={isSubmitting}
|
||||
>
|
||||
Open dialogue editor
|
||||
</Button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{usingReferences && useGuidedPrompt ? (
|
||||
/* Six-section format from the model's reference prompt guide. */
|
||||
<>
|
||||
@@ -873,12 +1006,31 @@ export default function CreateJobModal({
|
||||
>
|
||||
<Textarea
|
||||
id={`modal-prompt-${section}`}
|
||||
ref={(el) => {
|
||||
tagFieldRefs.current[section] = el;
|
||||
}}
|
||||
value={promptFields[section]}
|
||||
onChange={(e) => setPromptField(section, e.target.value)}
|
||||
onFocus={() => setActiveTagField(section)}
|
||||
rows={section === 'detailed_description' ? 5 : 2}
|
||||
placeholder={H3_SECTION_HINTS[section]}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
{section === 'detailed_description' && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="self-start"
|
||||
onClick={() => {
|
||||
setShotListMode('sections');
|
||||
setShotListOpen(true);
|
||||
}}
|
||||
disabled={isSubmitting}
|
||||
>
|
||||
Open shot list
|
||||
</Button>
|
||||
)}
|
||||
</FieldRow>
|
||||
))}
|
||||
<button
|
||||
@@ -898,8 +1050,12 @@ export default function CreateJobModal({
|
||||
>
|
||||
<Textarea
|
||||
id="modal-prompt"
|
||||
ref={(el) => {
|
||||
tagFieldRefs.current.prompt = el;
|
||||
}}
|
||||
value={prompt}
|
||||
onChange={(e) => setPrompt(e.target.value)}
|
||||
onFocus={() => setActiveTagField('prompt')}
|
||||
rows={isInference ? 3 : 2}
|
||||
placeholder={
|
||||
isInference
|
||||
@@ -909,6 +1065,21 @@ export default function CreateJobModal({
|
||||
required={!(usingReferences && useGuidedPrompt)}
|
||||
disabled={isSubmitting}
|
||||
/>
|
||||
{isH3Model && isInference && workloadType !== 't2i' && (
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="self-start"
|
||||
onClick={() => {
|
||||
setShotListMode('prompt');
|
||||
setShotListOpen(true);
|
||||
}}
|
||||
disabled={isSubmitting}
|
||||
>
|
||||
Open shot list
|
||||
</Button>
|
||||
)}
|
||||
</FieldRow>
|
||||
{usingReferences && (
|
||||
<button
|
||||
@@ -1384,6 +1555,38 @@ export default function CreateJobModal({
|
||||
</Button>
|
||||
</div>
|
||||
</form>
|
||||
|
||||
{/* Nested in the React tree (still portaled to <body>) so Radix treats
|
||||
clicks and focus inside these as inside this dialog. As siblings
|
||||
they counted as "outside", and closing one could close this too. */}
|
||||
<H3DialogueEditor
|
||||
open={dialogueEditorOpen}
|
||||
initialValue={getTagFieldValue(dialogueEditorField)}
|
||||
onApply={(value) => setTagFieldValue(dialogueEditorField, value)}
|
||||
onClose={() => setDialogueEditorOpen(false)}
|
||||
/>
|
||||
<H3ShotListEditor
|
||||
open={shotListOpen}
|
||||
subjectDefinitions={shotListMode === 'sections' ? promptFields.subject_definitions : promptSubjects}
|
||||
initialDetailedDescription={
|
||||
shotListMode === 'sections' ? promptFields.detailed_description : promptShots.body
|
||||
}
|
||||
initialRetentionAnalysis={
|
||||
shotListMode === 'sections' ? promptFields.retention_analysis : (promptRetention ?? '')
|
||||
}
|
||||
targetLabel={
|
||||
shotListMode === 'sections'
|
||||
? undefined
|
||||
: promptShots.field === 'integrated_multimodal_description'
|
||||
? 'integrated_multimodal_description'
|
||||
: promptShots.field === 'detailed_description'
|
||||
? 'Detailed description'
|
||||
: 'the prompt'
|
||||
}
|
||||
withRetention={shotListMode === 'sections' || promptSyncsRetention}
|
||||
onApply={shotListMode === 'sections' ? applyShotList : applyShotListToPrompt}
|
||||
onClose={() => setShotListOpen(false)}
|
||||
/>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,346 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { H3DialoguePreview } from '@/components/jobs/H3DialoguePreview';
|
||||
import { H3TagPalette } from '@/components/jobs/H3TagPalette';
|
||||
import { useUndoableText } from '@/hooks/useUndoableText';
|
||||
import { applyH3Preset, applyH3Tag, type H3Preset, type H3Tag } from '@/lib/h3Tags';
|
||||
import {
|
||||
SPEECH_RATES,
|
||||
dialogueTree,
|
||||
estimateDuration,
|
||||
type DialogueNode,
|
||||
type SpeechRate,
|
||||
} from '@/lib/h3DialogueEstimate';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
export interface H3DialogueEditorProps {
|
||||
open: boolean;
|
||||
/** Seeded from whichever prompt field the editor was opened for. */
|
||||
initialValue: string;
|
||||
onApply: (value: string) => void;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
/** Every `<d>...</d>` block's inner text -- what's actually spoken, ignoring surrounding shot/camera prose. */
|
||||
function extractDialogueBlocks(text: string): string[] {
|
||||
const blocks: string[] = [];
|
||||
const re = /<d>([\s\S]*?)<\/d>/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text))) blocks.push(m[1].replace(/^\s*\[[^\]]*\]\s*/, '')); // drop a leading [Language] marker
|
||||
return blocks;
|
||||
}
|
||||
|
||||
/**
|
||||
* Full-screen dialogue editor for MiniMax-H3 speech tags: the same tag
|
||||
* registry as `H3TagConsole`, plus a duration estimate (scoped to `<d>`
|
||||
* blocks so scene-description prose doesn't inflate it), a styled
|
||||
* performance preview, and emotion presets. Opens with a copy of the target
|
||||
* field's text; `onApply` hands the edited text back, `onClose` discards it.
|
||||
* The shot list editor reuses the same palette, preview, and undo history.
|
||||
*/
|
||||
export function H3DialogueEditor({ open, initialValue, onApply, onClose }: H3DialogueEditorProps) {
|
||||
const { text, type, commit, undo, redo, reset, canUndo, canRedo } = useUndoableText(initialValue);
|
||||
const [pace, setPace] = React.useState<SpeechRate>('Natural');
|
||||
const [calib, setCalib] = React.useState(1);
|
||||
const [actualSeconds, setActualSeconds] = React.useState('');
|
||||
const taRef = React.useRef<HTMLTextAreaElement>(null);
|
||||
const pending = React.useRef<[number, number] | null>(null);
|
||||
|
||||
function restoreCaretAtEnd(restored: string | null) {
|
||||
if (restored != null) pending.current = [restored.length, restored.length];
|
||||
}
|
||||
|
||||
// Re-seed from the field's current value each time the editor opens, the
|
||||
// same "seed on the open transition only" guard CreateJobModal itself
|
||||
// uses -- otherwise an in-progress edit here could be clobbered by a
|
||||
// stray parent re-render while open. History resets with it: undo
|
||||
// shouldn't reach back into a previous time this editor was opened.
|
||||
const wasOpenRef = React.useRef(false);
|
||||
React.useEffect(() => {
|
||||
if (open && !wasOpenRef.current) {
|
||||
reset(initialValue);
|
||||
setActualSeconds('');
|
||||
setCalib(1);
|
||||
}
|
||||
wasOpenRef.current = open;
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [open]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (pending.current && taRef.current) {
|
||||
const [s, e] = pending.current;
|
||||
taRef.current.focus();
|
||||
taRef.current.setSelectionRange(s, e);
|
||||
pending.current = null;
|
||||
}
|
||||
});
|
||||
|
||||
function handleKeyDown(e: React.KeyboardEvent<HTMLTextAreaElement>) {
|
||||
if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'z') return;
|
||||
e.preventDefault();
|
||||
restoreCaretAtEnd(e.shiftKey ? redo() : undo());
|
||||
}
|
||||
|
||||
function insertTag(tag: H3Tag) {
|
||||
const ta = taRef.current;
|
||||
const result = applyH3Tag(text, ta?.selectionStart ?? text.length, ta?.selectionEnd ?? text.length, tag);
|
||||
commit(result.value);
|
||||
pending.current = [result.start, result.end];
|
||||
}
|
||||
|
||||
function applyPreset(p: H3Preset) {
|
||||
const ta = taRef.current;
|
||||
const result = applyH3Preset(text, ta?.selectionStart ?? text.length, ta?.selectionEnd ?? text.length, p);
|
||||
commit(result.value);
|
||||
pending.current = [result.start, result.end];
|
||||
}
|
||||
|
||||
const dialogueBlocks = React.useMemo(() => extractDialogueBlocks(text), [text]);
|
||||
// Falls back to the whole field only when it contains no <d> blocks at
|
||||
// all yet -- e.g. a fresh field the user is about to write dialogue into.
|
||||
const segments = dialogueBlocks.length > 0 ? dialogueBlocks : [text];
|
||||
const estimates = React.useMemo(
|
||||
() => segments.map((s) => estimateDuration(s, SPEECH_RATES[pace], calib)),
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
[segments.join('\n'), pace, calib],
|
||||
);
|
||||
const totalSeconds = estimates.reduce((sum, e) => sum + e.total, 0);
|
||||
|
||||
function calibrate() {
|
||||
const actual = parseFloat(actualSeconds);
|
||||
if (!actual || actual <= 0 || segments.length === 0) return;
|
||||
const base = segments.reduce((sum, s) => sum + estimateDuration(s, SPEECH_RATES[pace], 1).total, 0);
|
||||
if (base > 0) setCalib(actual / base);
|
||||
}
|
||||
|
||||
/* ---- rough browser-voice preview, for rhythm/pacing only ---- */
|
||||
const synth = typeof window !== 'undefined' ? window.speechSynthesis : null;
|
||||
const [playing, setPlaying] = React.useState(false);
|
||||
const playTokenRef = React.useRef({ cancelled: true });
|
||||
|
||||
function stopPreview() {
|
||||
playTokenRef.current.cancelled = true;
|
||||
synth?.cancel();
|
||||
setPlaying(false);
|
||||
}
|
||||
|
||||
function playPreview() {
|
||||
if (!synth) return;
|
||||
stopPreview();
|
||||
const token = { cancelled: false };
|
||||
playTokenRef.current = token;
|
||||
setPlaying(true);
|
||||
const VOX: Record<string, string> = {
|
||||
laughs: 'ha ha ha', chuckle: 'heh', sighs: 'haah', gasp: 'ah', uh: 'uh', mhm: 'mhm', phew: 'phew',
|
||||
};
|
||||
const atoms: { say?: string; gap?: number }[] = [];
|
||||
for (const [i, seg] of segments.entries()) {
|
||||
if (i > 0) atoms.push({ gap: 0.6 });
|
||||
const walk = (nodes: DialogueNode[]) => {
|
||||
for (const n of nodes) {
|
||||
if (n.t === 'text') {
|
||||
const t = n.v.replace(/\s+/g, ' ').trim();
|
||||
if (t) atoms.push({ say: t });
|
||||
} else if (n.t === 'point') {
|
||||
atoms.push(VOX[n.name] ? { say: VOX[n.name] } : { gap: 0.25 });
|
||||
} else {
|
||||
if (n.name === 'humming') atoms.push({ say: 'hmm hmm' });
|
||||
else walk(n.children);
|
||||
}
|
||||
}
|
||||
};
|
||||
walk(dialogueTree(seg));
|
||||
}
|
||||
let i = 0;
|
||||
const step = () => {
|
||||
if (token.cancelled) return;
|
||||
if (i >= atoms.length) {
|
||||
setPlaying(false);
|
||||
return;
|
||||
}
|
||||
const a = atoms[i++];
|
||||
if (a.gap != null) {
|
||||
setTimeout(() => { if (!token.cancelled) step(); }, a.gap * 1000);
|
||||
return;
|
||||
}
|
||||
const u = new SpeechSynthesisUtterance(a.say);
|
||||
u.rate = 0.95;
|
||||
u.onend = () => { if (!token.cancelled) step(); };
|
||||
u.onerror = () => { if (!token.cancelled) step(); };
|
||||
synth.speak(u);
|
||||
};
|
||||
step();
|
||||
}
|
||||
|
||||
React.useEffect(() => () => stopPreview(), []); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
function handleApply() {
|
||||
onApply(text);
|
||||
onClose();
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={(o) => { if (!o) onClose(); }}>
|
||||
<DialogContent className="flex h-[92vh] max-h-[920px] w-[96vw] max-w-[1180px] flex-col gap-0 overflow-hidden p-0">
|
||||
<DialogHeader className="border-b border-border px-5 py-4">
|
||||
<DialogTitle>Dialogue editor</DialogTitle>
|
||||
<DialogDescription>
|
||||
Click a tag to insert it at the cursor, or select text first to wrap it. Applying writes the edited text
|
||||
back into the field you opened this from.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="flex min-h-0 flex-1">
|
||||
{/* palette */}
|
||||
<H3TagPalette
|
||||
onInsertTag={insertTag}
|
||||
onApplyPreset={applyPreset}
|
||||
className="w-64 shrink-0 overflow-y-auto border-r border-border p-3"
|
||||
/>
|
||||
|
||||
{/* editor + preview */}
|
||||
<div className="flex min-w-0 flex-1 flex-col gap-4 overflow-y-auto p-4">
|
||||
{/* duration estimate */}
|
||||
<div className="rounded-xl border border-border bg-secondary/30 p-4">
|
||||
<div className="flex flex-wrap items-baseline justify-between gap-3">
|
||||
<div className="flex items-baseline gap-3">
|
||||
<span className="text-2xl font-semibold tracking-tight text-foreground">
|
||||
{totalSeconds.toFixed(1)}<span className="ml-1 text-sm font-normal text-muted-foreground">s spoken</span>
|
||||
</span>
|
||||
<span className="font-mono text-xs text-muted-foreground">
|
||||
{dialogueBlocks.length > 0
|
||||
? `across ${dialogueBlocks.length} <d> block${dialogueBlocks.length === 1 ? '' : 's'}`
|
||||
: 'no <d> blocks yet — estimating the whole field'}
|
||||
</span>
|
||||
</div>
|
||||
<div className="flex items-center gap-1 rounded-lg border border-border bg-card p-1">
|
||||
{(Object.keys(SPEECH_RATES) as SpeechRate[]).map((p) => (
|
||||
<button
|
||||
key={p}
|
||||
type="button"
|
||||
onClick={() => setPace(p)}
|
||||
className={cn(
|
||||
'rounded-md px-2.5 py-1 text-xs transition-colors',
|
||||
pace === p ? 'bg-secondary text-foreground' : 'text-muted-foreground hover:bg-accent',
|
||||
)}
|
||||
>
|
||||
{p}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{segments.length > 1 && (
|
||||
<div className="mt-3 flex flex-wrap gap-1.5">
|
||||
{estimates.map((e, i) => (
|
||||
<span key={i} className="rounded-md border border-border bg-card px-2 py-0.5 font-mono text-[11px] text-muted-foreground">
|
||||
block {i + 1}: {e.total.toFixed(1)}s
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="mt-3 flex flex-wrap items-center gap-2 text-xs text-muted-foreground">
|
||||
<span title="Enter the real length of a clip you already generated, to tune the estimate to your setup">
|
||||
calibrate
|
||||
</span>
|
||||
<input
|
||||
value={actualSeconds}
|
||||
onChange={(e) => setActualSeconds(e.target.value)}
|
||||
placeholder="actual s"
|
||||
inputMode="decimal"
|
||||
className="w-16 rounded-md border border-input bg-card px-2 py-1 text-foreground"
|
||||
/>
|
||||
<Button type="button" variant="outline" size="sm" onClick={calibrate}>Set</Button>
|
||||
{Math.abs(calib - 1) > 0.001 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => { setCalib(1); setActualSeconds(''); }}
|
||||
className="font-mono text-muted-foreground/80 hover:text-foreground"
|
||||
title="Reset calibration"
|
||||
>
|
||||
×{calib.toFixed(2)}
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* audio preview */}
|
||||
{synth && (
|
||||
<div className="flex items-center gap-3 rounded-xl border border-border bg-secondary/30 p-3">
|
||||
<Button type="button" size="sm" onClick={playing ? stopPreview : playPreview}>
|
||||
{playing ? '■ Stop' : '▶ Play preview'}
|
||||
</Button>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
A stand-in browser voice for rhythm only — it won't match MiniMax's voice.
|
||||
</span>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* the field itself */}
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-semibold text-foreground">Text</span>
|
||||
<div className="flex items-center gap-1">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => restoreCaretAtEnd(undo())}
|
||||
disabled={!canUndo}
|
||||
title="Undo (⌘/Ctrl+Z)"
|
||||
>
|
||||
↶ Undo
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => restoreCaretAtEnd(redo())}
|
||||
disabled={!canRedo}
|
||||
title="Redo (⌘/Ctrl+Shift+Z)"
|
||||
>
|
||||
↷ Redo
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
<Textarea
|
||||
ref={taRef}
|
||||
value={text}
|
||||
onChange={(e) => type(e.target.value)}
|
||||
onKeyDown={handleKeyDown}
|
||||
spellCheck={false}
|
||||
rows={8}
|
||||
className="flex-1 font-mono text-sm"
|
||||
/>
|
||||
|
||||
{/* performance preview */}
|
||||
<div>
|
||||
<div className="mb-2 text-xs font-semibold text-foreground">Performance preview</div>
|
||||
<H3DialoguePreview
|
||||
text={text}
|
||||
className="rounded-xl border border-border bg-card p-4 text-sm leading-7 text-foreground"
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<DialogFooter className="border-t border-border px-5 py-3">
|
||||
<Button type="button" variant="outline" onClick={onClose}>Cancel</Button>
|
||||
<Button type="button" onClick={handleApply}>Apply to field</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,44 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { CATEGORY_STYLE } from '@/components/jobs/H3TagPalette';
|
||||
import { dialogueTree, type DialogueNode } from '@/lib/h3DialogueEstimate';
|
||||
import { H3_TAG_CATEGORIES } from '@/lib/h3Tags';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
const CATEGORY_BY_TAG_NAME: Record<string, string> = Object.fromEntries(
|
||||
H3_TAG_CATEGORIES.flatMap((c) => c.tags.map((t) => [t.open.replace(/[<>]/g, ''), c.id])),
|
||||
);
|
||||
|
||||
function renderPreviewNodes(nodes: DialogueNode[], keyPrefix = ''): React.ReactNode[] {
|
||||
return nodes.map((n, i) => {
|
||||
const key = `${keyPrefix}${i}`;
|
||||
if (n.t === 'text') return <span key={key}>{n.v}</span>;
|
||||
if (n.t === 'point') {
|
||||
const cat = CATEGORY_BY_TAG_NAME[n.name] ?? 'body';
|
||||
const style = CATEGORY_STYLE[cat] ?? CATEGORY_STYLE.body;
|
||||
return (
|
||||
<span
|
||||
key={key}
|
||||
className={cn(
|
||||
'mx-0.5 inline-block rounded border px-1.5 py-0.5 font-mono text-[11px] leading-none',
|
||||
style.chip,
|
||||
)}
|
||||
>
|
||||
{n.name}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
const inner = renderPreviewNodes(n.children, `${key}-`);
|
||||
if (n.name === 'i') return <strong key={key} className="italic">{inner}</strong>;
|
||||
if (n.name === 'whisper') return <span key={key} className="text-[0.92em] tracking-wide opacity-60">{inner}</span>;
|
||||
if (n.name === 'humming') return <em key={key} className="underline decoration-wavy decoration-emerald-500 underline-offset-4">{inner}</em>;
|
||||
return <span key={key}>{inner}</span>;
|
||||
});
|
||||
}
|
||||
|
||||
/** Styled "performance" rendering of speech tags: chips for point tags, styling for wrapping ones. */
|
||||
export function H3DialoguePreview({ text, className }: { text: string; className?: string }) {
|
||||
return <div className={className}>{renderPreviewNodes(dialogueTree(text))}</div>;
|
||||
}
|
||||
@@ -0,0 +1,184 @@
|
||||
import { render, screen, within } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { H3DialogueEditor } from './H3DialogueEditor';
|
||||
import { H3ShotListEditor } from './H3ShotListEditor';
|
||||
|
||||
const SUBJECTS = '<Subject 1> is the courier.\n<Subject 2> is the rival.';
|
||||
const DETAILED = [
|
||||
'[Shot 1] Wide shot. A rooftop at night.',
|
||||
'[Shot 2] Close-up shot, handheld. He turns. (S2) <d>[English] I had no choice.</d>',
|
||||
].join('\n\n');
|
||||
|
||||
describe('H3DialogueEditor (shared palette + undo)', () => {
|
||||
it('inserts a tag at the cursor, undoes it, and applies the edited text', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = vi.fn();
|
||||
render(<H3DialogueEditor open initialValue="Hello there" onApply={onApply} onClose={() => {}} />);
|
||||
|
||||
const textarea = screen.getByDisplayValue('Hello there') as HTMLTextAreaElement;
|
||||
textarea.focus();
|
||||
textarea.setSelectionRange(5, 5);
|
||||
await user.click(screen.getByRole('button', { name: /<pause>/ }));
|
||||
expect(textarea.value).toBe('Hello <pause> there');
|
||||
|
||||
await user.click(screen.getByRole('button', { name: /Undo/ }));
|
||||
expect(textarea.value).toBe('Hello there');
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Nervous' }));
|
||||
expect(textarea.value).toContain('<uh> <stutter>');
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Apply to field' }));
|
||||
expect(onApply).toHaveBeenCalledWith(expect.stringContaining('<uh> <stutter>'));
|
||||
});
|
||||
});
|
||||
|
||||
describe('H3ShotListEditor dialogue tooling', () => {
|
||||
function renderEditor(onApply = vi.fn()) {
|
||||
render(
|
||||
<H3ShotListEditor
|
||||
open
|
||||
subjectDefinitions={SUBJECTS}
|
||||
initialDetailedDescription={DETAILED}
|
||||
initialRetentionAnalysis=""
|
||||
onApply={onApply}
|
||||
onClose={() => {}}
|
||||
/>,
|
||||
);
|
||||
return onApply;
|
||||
}
|
||||
|
||||
it('disables the palette until a shot dialogue is focused', () => {
|
||||
renderEditor();
|
||||
expect(screen.getByText(/Click into a shot's dialogue/)).toBeInTheDocument();
|
||||
expect(screen.getByRole('button', { name: /<pause>/ })).toBeDisabled();
|
||||
});
|
||||
|
||||
it('inserts a palette tag into the focused shot dialogue and writes it on apply', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = renderEditor();
|
||||
|
||||
const dialogueBoxes = screen.getAllByPlaceholderText('What the speaker says') as HTMLTextAreaElement[];
|
||||
expect(dialogueBoxes).toHaveLength(1);
|
||||
expect(dialogueBoxes[0].value).toBe('I had no choice.');
|
||||
|
||||
await user.click(dialogueBoxes[0]);
|
||||
dialogueBoxes[0].setSelectionRange(1, 1);
|
||||
expect(screen.getByText('Tags insert into Shot 2 dialogue')).toBeInTheDocument();
|
||||
|
||||
await user.click(screen.getByRole('button', { name: /<pause>/ }));
|
||||
expect(dialogueBoxes[0].value).toBe('I <pause> had no choice.');
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
expect(onApply).toHaveBeenCalledTimes(1);
|
||||
expect(onApply.mock.calls[0][0]).toContain('<d>[English] I <pause> had no choice.</d>');
|
||||
});
|
||||
|
||||
it('adds and removes dialogue lines, writing every line on apply', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = renderEditor();
|
||||
|
||||
await user.click(screen.getAllByRole('button', { name: '+ Add line' })[1]);
|
||||
const boxes = screen.getAllByPlaceholderText('What the speaker says') as HTMLTextAreaElement[];
|
||||
await user.type(boxes[1], 'Then go.');
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
expect(onApply.mock.calls[0][0]).toContain(
|
||||
'(S2) <d>[English] I had no choice.</d> (S2) <d>[English] Then go.</d>',
|
||||
);
|
||||
});
|
||||
|
||||
it('shows the styled preview once a shot dialogue contains tags', async () => {
|
||||
const user = userEvent.setup();
|
||||
renderEditor();
|
||||
const boxes = screen.getAllByPlaceholderText('What the speaker says') as HTMLTextAreaElement[];
|
||||
await user.click(boxes[0]);
|
||||
await user.click(screen.getByRole('button', { name: /<long pause>/ }));
|
||||
|
||||
const shot2 = boxes[0].closest('[id^="shot-card-"]') as HTMLElement;
|
||||
expect(within(shot2).getByText('long pause')).toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
||||
describe('H3ShotListEditor ordered body', () => {
|
||||
function renderWith(detailed: string) {
|
||||
const onApply = vi.fn();
|
||||
render(
|
||||
<H3ShotListEditor
|
||||
open
|
||||
subjectDefinitions={SUBJECTS}
|
||||
initialDetailedDescription={detailed}
|
||||
initialRetentionAnalysis=""
|
||||
onApply={onApply}
|
||||
onClose={() => {}}
|
||||
/>,
|
||||
);
|
||||
return onApply;
|
||||
}
|
||||
|
||||
it('shows hand-written dialogue as an editable line between its prose and applies the text unchanged', async () => {
|
||||
const user = userEvent.setup();
|
||||
const text = "[Shot 1] He (S1) says: <d>[English] I think you're sorry.</d> <cutoff>";
|
||||
const onApply = renderWith(text);
|
||||
|
||||
expect(screen.getByText(/keeps its original wording/)).toBeInTheDocument();
|
||||
const lines = screen.getAllByPlaceholderText('What the speaker says') as HTMLTextAreaElement[];
|
||||
expect(lines).toHaveLength(1);
|
||||
expect(lines[0].value).toBe("I think you're sorry.");
|
||||
const prose = screen.getAllByPlaceholderText("What's happening in this shot") as HTMLTextAreaElement[];
|
||||
expect(prose.map((p) => p.value)).toEqual(['He (S1) says: ', ' <cutoff>']);
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
expect(onApply.mock.calls[0][0]).toBe(text);
|
||||
});
|
||||
|
||||
it('editing a hand-written line keeps the tag after it where it was', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = renderWith("[Shot 1] He (S1) says: <d>[English] Sorry.</d> <cutoff>");
|
||||
const [line] = screen.getAllByPlaceholderText('What the speaker says') as HTMLTextAreaElement[];
|
||||
await user.type(line, ' Really.');
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
expect(onApply.mock.calls[0][0]).toBe("[Shot 1] He (S1) says: <d>[English] Sorry. Really.</d> <cutoff>");
|
||||
});
|
||||
|
||||
it('reorders dialogue lines with the move buttons', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = renderWith('[Shot 1] Wide shot. (S1) <d>[English] One.</d> (S2) <d>[English] Two.</d>');
|
||||
await user.click(screen.getByRole('button', { name: 'Move line 2 up' }));
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
expect(onApply.mock.calls[0][0]).toBe(
|
||||
'[Shot 1] Wide shot. (S2) <d>[English] Two.</d> (S1) <d>[English] One.</d>',
|
||||
);
|
||||
});
|
||||
|
||||
it('adds a text block after the existing content', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = renderWith('[Shot 1] Wide shot. A dock.');
|
||||
await user.click(screen.getByRole('button', { name: '+ Add text' }));
|
||||
const prose = screen.getAllByPlaceholderText("What's happening in this shot") as HTMLTextAreaElement[];
|
||||
expect(prose).toHaveLength(2);
|
||||
await user.type(prose[1], 'Rain falls.');
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
expect(onApply.mock.calls[0][0]).toBe('[Shot 1] Wide shot. A dock. Rain falls.');
|
||||
});
|
||||
|
||||
it('removes a text block', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = renderWith('[Shot 1] Wide shot. A dock. (S1) <d>[English] Hi.</d>');
|
||||
await user.click(screen.getByRole('button', { name: 'Remove text 1' }));
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
expect(onApply.mock.calls[0][0]).toBe('[Shot 1] Wide shot. (S1) <d>[English] Hi.</d>');
|
||||
});
|
||||
|
||||
it('adds a new line at the end of the shot, after any closing prose', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = renderWith('[Shot 1] Wide shot. He speaks. (S1) <d>[English] First.</d> He pauses.');
|
||||
await user.click(screen.getByRole('button', { name: '+ Add line' }));
|
||||
const lines = screen.getAllByPlaceholderText('What the speaker says') as HTMLTextAreaElement[];
|
||||
await user.type(lines[1], 'Later.');
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
expect(onApply.mock.calls[0][0]).toBe(
|
||||
'[Shot 1] Wide shot. He speaks. (S1) <d>[English] First.</d> He pauses. (S1) <d>[English] Later.</d>',
|
||||
);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,123 @@
|
||||
import { render, screen, within } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { H3ShotListEditor } from './H3ShotListEditor';
|
||||
|
||||
// Original test material -- not from any real screenplay.
|
||||
const SCRIPT = [
|
||||
'CAPTAIN: Sit down. (EXHALES) We need to talk about the ledger.',
|
||||
'',
|
||||
'CLERK: Yes, sir.',
|
||||
'',
|
||||
'CAPTAIN: Three crates are missing and nobody on this dock saw a thing.',
|
||||
].join('\n');
|
||||
|
||||
const SUBJECTS = '<Subject 1> is the captain.\n<Subject 2> is the clerk.';
|
||||
|
||||
function renderEditor(detailed = '', onApply = vi.fn()) {
|
||||
render(
|
||||
<H3ShotListEditor
|
||||
open
|
||||
subjectDefinitions={SUBJECTS}
|
||||
initialDetailedDescription={detailed}
|
||||
initialRetentionAnalysis=""
|
||||
onApply={onApply}
|
||||
onClose={() => {}}
|
||||
/>,
|
||||
);
|
||||
return onApply;
|
||||
}
|
||||
|
||||
async function importScript(user: ReturnType<typeof userEvent.setup>) {
|
||||
await user.click(screen.getByRole('button', { name: 'Import script…' }));
|
||||
const dialog = await screen.findByRole('dialog', { name: 'Import script' });
|
||||
await user.click(within(dialog).getByLabelText('Script'));
|
||||
await user.paste(SCRIPT);
|
||||
return dialog;
|
||||
}
|
||||
|
||||
describe('script import', () => {
|
||||
it('maps speakers to subjects by default and previews the clip', async () => {
|
||||
const user = userEvent.setup();
|
||||
renderEditor();
|
||||
const dialog = await importScript(user);
|
||||
|
||||
expect(within(dialog).getByLabelText('CAPTAIN subject')).toHaveValue('<Subject 1>');
|
||||
expect(within(dialog).getByLabelText('CLERK subject')).toHaveValue('<Subject 2>');
|
||||
expect(within(dialog).getByLabelText('CAPTAIN speaker tag')).toHaveValue('S1');
|
||||
expect(within(dialog).getByLabelText('CLERK speaker tag')).toHaveValue('S2');
|
||||
expect(within(dialog).getByRole('option', { name: /Clip 1 of \d+/ })).toBeInTheDocument();
|
||||
expect(within(dialog).getByText('Establishing')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('loads the chosen clip into the shot list and writes it on apply', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onApply = renderEditor();
|
||||
const dialog = await importScript(user);
|
||||
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Load clip' }));
|
||||
|
||||
const boxes = (await screen.findAllByPlaceholderText('What the speaker says')) as HTMLTextAreaElement[];
|
||||
expect(boxes[0].value).toBe('Sit down. <exhale> We need to talk about the ledger.');
|
||||
expect(boxes[1].value).toBe('Yes, sir.');
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
const [detailed, retention] = onApply.mock.calls[0];
|
||||
expect(detailed).toContain('[Shot 1] Establishing shot.');
|
||||
expect(detailed).toContain('(S1) <d>[English] Sit down. <exhale> We need to talk about the ledger.</d> (S2) <d>[English] Yes, sir.</d>');
|
||||
expect(retention).toContain('<Subject 1> (appears in');
|
||||
expect(retention).toContain('<Subject 2> (appears in');
|
||||
});
|
||||
|
||||
it('warns that loading replaces the shots already in the list', async () => {
|
||||
const user = userEvent.setup();
|
||||
renderEditor('[Shot 1] Wide shot. A dock.\n\n[Shot 2] Close-up shot. A face.');
|
||||
const dialog = await importScript(user);
|
||||
expect(within(dialog).getByText('Replaces the 2 shots currently in the list.')).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('reloading the same clip after edits restores the imported text', async () => {
|
||||
const user = userEvent.setup();
|
||||
renderEditor();
|
||||
let dialog = await importScript(user);
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Load clip' }));
|
||||
|
||||
let boxes = (await screen.findAllByPlaceholderText('What the speaker says')) as HTMLTextAreaElement[];
|
||||
await user.type(boxes[1], ' EDITED');
|
||||
expect(boxes[1].value).toBe('Yes, sir. EDITED');
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Import script…' }));
|
||||
dialog = await screen.findByRole('dialog', { name: 'Import script' });
|
||||
// Loading a clip advances the picker to the next one; choose clip 1 again.
|
||||
await user.selectOptions(within(dialog).getByLabelText('Clip to load'), '0');
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Load clip' }));
|
||||
|
||||
boxes = (await screen.findAllByPlaceholderText('What the speaker says')) as HTMLTextAreaElement[];
|
||||
expect(boxes[1].value).toBe('Yes, sir.');
|
||||
});
|
||||
|
||||
it('advances to the next clip after loading one', async () => {
|
||||
const user = userEvent.setup();
|
||||
renderEditor();
|
||||
let dialog = await importScript(user);
|
||||
const picker = within(dialog).getByLabelText('Clip to load') as HTMLSelectElement;
|
||||
expect(picker.options.length).toBeGreaterThan(1);
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Load clip' }));
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Import script…' }));
|
||||
dialog = await screen.findByRole('dialog', { name: 'Import script' });
|
||||
expect((within(dialog).getByLabelText('Clip to load') as HTMLSelectElement).value).toBe('1');
|
||||
});
|
||||
|
||||
it('explains when the text has no speaker turns', async () => {
|
||||
const user = userEvent.setup();
|
||||
renderEditor();
|
||||
await user.click(screen.getByRole('button', { name: 'Import script…' }));
|
||||
const dialog = await screen.findByRole('dialog', { name: 'Import script' });
|
||||
await user.click(within(dialog).getByLabelText('Script'));
|
||||
await user.paste('just some prose');
|
||||
expect(within(dialog).getByRole('alert')).toHaveTextContent(/No "NAME: line" turns found/);
|
||||
expect(within(dialog).getByRole('button', { name: 'Load clip' })).toBeDisabled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,247 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { NativeSelect } from '@/components/ui/native-select';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { buildClips, parseScript, scriptSpeakers, type ImportSpeaker } from '@/lib/h3ScriptImport';
|
||||
import { SHOT_TYPES, shotLines, shotProse, type Shot } from '@/lib/h3Shots';
|
||||
|
||||
const TYPE_LABEL = Object.fromEntries(SHOT_TYPES.map((t) => [t.id, t.label]));
|
||||
const DEFAULT_CLIP_SECONDS = 11;
|
||||
|
||||
export interface H3ScriptImportDialogProps {
|
||||
open: boolean;
|
||||
subjectLabels: string[];
|
||||
/** shots currently in the list -- loading a clip replaces them */
|
||||
existingShotCount: number;
|
||||
onLoad: (shots: Shot[]) => void;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
function preview(shot: Shot): string {
|
||||
const line = shotLines(shot).find((l) => l.text.trim());
|
||||
const text = line ? `${line.speaker ? `${line.speaker}: ` : ''}“${line.text.trim()}”` : shotProse(shot);
|
||||
return text.length > 90 ? `${text.slice(0, 89)}…` : text;
|
||||
}
|
||||
|
||||
/**
|
||||
* Paste (or load) a "NAME: line" script and pick one clip-sized chunk of it
|
||||
* to load into the shot list. The script and speaker mapping persist between
|
||||
* openings, so working through a long scene is: load clip, apply, next clip.
|
||||
*/
|
||||
export function H3ScriptImportDialog({
|
||||
open,
|
||||
subjectLabels,
|
||||
existingShotCount,
|
||||
onLoad,
|
||||
onClose,
|
||||
}: H3ScriptImportDialogProps) {
|
||||
const [script, setScript] = React.useState('');
|
||||
const [clipSecondsInput, setClipSecondsInput] = React.useState(String(DEFAULT_CLIP_SECONDS));
|
||||
const [overrides, setOverrides] = React.useState<Record<string, Partial<ImportSpeaker>>>({});
|
||||
const [clipIndex, setClipIndex] = React.useState(0);
|
||||
|
||||
const turns = React.useMemo(() => parseScript(script), [script]);
|
||||
const names = React.useMemo(() => scriptSpeakers(turns), [turns]);
|
||||
|
||||
const speakers = React.useMemo(() => {
|
||||
const map: Record<string, ImportSpeaker> = {};
|
||||
names.forEach((name, i) => {
|
||||
const o = overrides[name] ?? {};
|
||||
map[name] = {
|
||||
subject: 'subject' in o ? (o.subject ?? null) : (subjectLabels[i] ?? null),
|
||||
tag: o.tag ?? `S${i + 1}`,
|
||||
};
|
||||
});
|
||||
return map;
|
||||
}, [names, overrides, subjectLabels]);
|
||||
|
||||
const parsedSeconds = Number(clipSecondsInput);
|
||||
const targetSeconds = Number.isFinite(parsedSeconds) ? Math.min(Math.max(parsedSeconds, 4), 12) : DEFAULT_CLIP_SECONDS;
|
||||
|
||||
const clips = React.useMemo(
|
||||
() => (turns.length ? buildClips(turns, { speakers, targetSeconds }) : []),
|
||||
[turns, speakers, targetSeconds],
|
||||
);
|
||||
const selected = Math.min(clipIndex, Math.max(clips.length - 1, 0));
|
||||
const clip = clips[selected];
|
||||
|
||||
function setOverride(name: string, patch: Partial<ImportSpeaker>) {
|
||||
setOverrides((prev) => ({ ...prev, [name]: { ...prev[name], ...patch } }));
|
||||
}
|
||||
|
||||
async function handleFile(e: React.ChangeEvent<HTMLInputElement>) {
|
||||
const file = e.target.files?.[0];
|
||||
e.target.value = '';
|
||||
if (!file) return;
|
||||
setScript(await file.text());
|
||||
setClipIndex(0);
|
||||
}
|
||||
|
||||
function handleLoad() {
|
||||
if (!clip) return;
|
||||
onLoad(clip.shots);
|
||||
setClipIndex(Math.min(selected + 1, clips.length - 1));
|
||||
onClose();
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={(o) => { if (!o) onClose(); }}>
|
||||
<DialogContent className="flex max-h-[90vh] w-[94vw] max-w-[760px] flex-col gap-0 overflow-hidden p-0">
|
||||
<DialogHeader className="border-b border-border px-5 py-4">
|
||||
<DialogTitle>Import script</DialogTitle>
|
||||
<DialogDescription>
|
||||
Paste a script of NAME: line turns. It's split into clip-sized chunks; load one at a time into the
|
||||
shot list, then set the camera and descriptions.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col gap-4 overflow-y-auto p-5">
|
||||
<div className="flex flex-col gap-2">
|
||||
<label htmlFor="script-import-text" className="text-xs font-semibold text-foreground">
|
||||
Script
|
||||
</label>
|
||||
<Textarea
|
||||
id="script-import-text"
|
||||
value={script}
|
||||
onChange={(e) => {
|
||||
setScript(e.target.value);
|
||||
setClipIndex(0);
|
||||
}}
|
||||
rows={8}
|
||||
spellCheck={false}
|
||||
placeholder={'MARK: First line...\n\nJORDAN: Reply...'}
|
||||
className="font-mono text-xs"
|
||||
/>
|
||||
<Input
|
||||
type="file"
|
||||
accept=".md,.txt,.text"
|
||||
aria-label="Load script file"
|
||||
onChange={handleFile}
|
||||
className="h-auto py-2 file:mr-3 file:cursor-pointer file:rounded-md file:border-0 file:bg-secondary file:px-2 file:py-1 file:text-sm file:text-secondary-foreground"
|
||||
/>
|
||||
</div>
|
||||
|
||||
{script.trim() && turns.length === 0 && (
|
||||
<p role="alert" className="text-sm text-destructive">
|
||||
No "NAME: line" turns found. Each turn should start with a speaker name and a colon.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{names.length > 0 && (
|
||||
<div className="flex flex-col gap-2">
|
||||
<span className="text-xs font-semibold text-foreground">Speakers</span>
|
||||
{names.map((name) => (
|
||||
<div key={name} className="grid grid-cols-[1fr_1fr_5rem] items-center gap-2">
|
||||
<span className="truncate text-sm text-foreground">{name}</span>
|
||||
<NativeSelect
|
||||
aria-label={`${name} subject`}
|
||||
value={speakers[name].subject ?? ''}
|
||||
onChange={(e) => setOverride(name, { subject: e.target.value || null })}
|
||||
>
|
||||
<option value="">Not a tracked subject</option>
|
||||
{subjectLabels.map((label) => (
|
||||
<option key={label} value={label}>{label}</option>
|
||||
))}
|
||||
</NativeSelect>
|
||||
<Input
|
||||
aria-label={`${name} speaker tag`}
|
||||
value={speakers[name].tag}
|
||||
onChange={(e) => setOverride(name, { tag: e.target.value })}
|
||||
className="font-mono"
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
{subjectLabels.length === 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
No subjects are defined yet, so speakers can't be linked. Add them under Subject definitions
|
||||
to track who appears in each shot.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{turns.length > 0 && (
|
||||
<>
|
||||
<label className="flex flex-col gap-1 text-xs text-muted-foreground">
|
||||
Clip length target (seconds)
|
||||
<Input
|
||||
type="number"
|
||||
min={4}
|
||||
max={12}
|
||||
step={1}
|
||||
value={clipSecondsInput}
|
||||
onChange={(e) => {
|
||||
setClipSecondsInput(e.target.value);
|
||||
setClipIndex(0);
|
||||
}}
|
||||
className="w-28"
|
||||
/>
|
||||
<span>One generation runs about 11 s (277 frames at 24 fps), so each clip becomes its own job.</span>
|
||||
</label>
|
||||
|
||||
<div className="flex flex-col gap-2">
|
||||
<label htmlFor="script-import-clip" className="text-xs font-semibold text-foreground">
|
||||
Clip to load
|
||||
</label>
|
||||
<NativeSelect
|
||||
id="script-import-clip"
|
||||
value={selected}
|
||||
onChange={(e) => setClipIndex(Number(e.target.value))}
|
||||
>
|
||||
{clips.map((c, i) => (
|
||||
<option key={i} value={i}>
|
||||
Clip {i + 1} of {clips.length} · {c.shots.length} shot{c.shots.length === 1 ? '' : 's'}
|
||||
{c.continuation ? ' · continues previous shot' : ''}
|
||||
</option>
|
||||
))}
|
||||
</NativeSelect>
|
||||
|
||||
{clip?.continuation && (
|
||||
<p className="rounded-lg border border-border bg-secondary/30 px-3 py-2 text-xs text-muted-foreground">
|
||||
This clip continues the previous clip's last shot, cut short because that speech didn't
|
||||
fit one generation. Generate it from the previous clip's last frame (as the start Image)
|
||||
with the same references.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{clip && (
|
||||
<ol className="flex flex-col gap-1 rounded-xl border border-border bg-secondary/20 p-3">
|
||||
{clip.shots.map((shot, i) => (
|
||||
<li key={shot.id} className="flex gap-2 text-xs">
|
||||
<span className="w-6 shrink-0 font-mono text-muted-foreground">{i + 1}</span>
|
||||
<span className="w-24 shrink-0 font-medium text-foreground">{TYPE_LABEL[shot.type]}</span>
|
||||
<span className="min-w-0 text-muted-foreground">{preview(shot)}</span>
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<DialogFooter className="items-center border-t border-border px-5 py-3">
|
||||
{existingShotCount > 0 && clip && (
|
||||
<span className="mr-auto text-xs text-muted-foreground">
|
||||
Replaces the {existingShotCount} shot{existingShotCount === 1 ? '' : 's'} currently in the list.
|
||||
</span>
|
||||
)}
|
||||
<Button type="button" variant="outline" onClick={onClose}>Cancel</Button>
|
||||
<Button type="button" onClick={handleLoad} disabled={!clip}>Load clip</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,92 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { H3DialoguePreview } from '@/components/jobs/H3DialoguePreview';
|
||||
import { applyH3Preset, applyH3Tag, type H3Preset, type H3Tag } from '@/lib/h3Tags';
|
||||
|
||||
/** What the shared tag palette drives: the last-focused shot dialogue field. */
|
||||
export interface H3ShotDialogueHandle {
|
||||
insertTag: (tag: H3Tag) => void;
|
||||
applyPreset: (preset: H3Preset) => void;
|
||||
}
|
||||
|
||||
export interface H3ShotDialogueFieldProps {
|
||||
/** The line's text. The parent owns it (and the undo history), so this is fully controlled. */
|
||||
value: string;
|
||||
/** Typing. */
|
||||
onChange: (value: string) => void;
|
||||
/** A tag or preset inserted from the palette -- a discrete edit, unlike typing. */
|
||||
onInsert: (value: string) => void;
|
||||
onFocus: () => void;
|
||||
active?: boolean;
|
||||
label?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* One spoken line: a textarea plus, once it contains tags, a styled preview.
|
||||
* The shared palette inserts at the cursor through the ref. Undo/redo isn't
|
||||
* here -- the shot list editor keeps one history for everything in it.
|
||||
*/
|
||||
export const H3ShotDialogueField = React.forwardRef<H3ShotDialogueHandle, H3ShotDialogueFieldProps>(
|
||||
function H3ShotDialogueField({ value, onChange, onInsert, onFocus, active, label = 'Dialogue' }, ref) {
|
||||
const taRef = React.useRef<HTMLTextAreaElement>(null);
|
||||
// Selection to restore once the inserted text has rendered.
|
||||
const pending = React.useRef<[number, number] | null>(null);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (pending.current && taRef.current) {
|
||||
const [s, e] = pending.current;
|
||||
taRef.current.focus();
|
||||
taRef.current.setSelectionRange(s, e);
|
||||
pending.current = null;
|
||||
}
|
||||
});
|
||||
|
||||
function selection(): [number, number] {
|
||||
const ta = taRef.current;
|
||||
return [ta?.selectionStart ?? value.length, ta?.selectionEnd ?? value.length];
|
||||
}
|
||||
|
||||
React.useImperativeHandle(ref, () => ({
|
||||
insertTag(tag) {
|
||||
const [s, e] = selection();
|
||||
const result = applyH3Tag(value, s, e, tag);
|
||||
pending.current = [result.start, result.end];
|
||||
onInsert(result.value);
|
||||
},
|
||||
applyPreset(preset) {
|
||||
const [s, e] = selection();
|
||||
const result = applyH3Preset(value, s, e, preset);
|
||||
pending.current = [result.start, result.end];
|
||||
onInsert(result.value);
|
||||
},
|
||||
}));
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-1">
|
||||
<div className="text-xs text-muted-foreground">
|
||||
{label}
|
||||
{active && <span className="ml-2 text-accent-blue">tags insert here</span>}
|
||||
</div>
|
||||
<Textarea
|
||||
ref={taRef}
|
||||
value={value}
|
||||
onChange={(e) => onChange(e.target.value)}
|
||||
onFocus={onFocus}
|
||||
rows={2}
|
||||
spellCheck={false}
|
||||
placeholder="What the speaker says"
|
||||
className="font-mono text-sm"
|
||||
/>
|
||||
{value.includes('<') && (
|
||||
<H3DialoguePreview
|
||||
text={value}
|
||||
className="rounded-lg border border-border bg-card px-3 py-2 text-sm leading-7 text-foreground"
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
},
|
||||
);
|
||||
@@ -0,0 +1,690 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { NativeSelect } from '@/components/ui/native-select';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import { H3ShotDialogueField, type H3ShotDialogueHandle } from '@/components/jobs/H3ShotDialogueField';
|
||||
import { H3ScriptImportDialog } from '@/components/jobs/H3ScriptImportDialog';
|
||||
import { H3ShotStoryboard } from '@/components/jobs/H3ShotStoryboard';
|
||||
import { H3TagPalette } from '@/components/jobs/H3TagPalette';
|
||||
import type { H3Preset, H3Tag } from '@/lib/h3Tags';
|
||||
import {
|
||||
CAMERA_MOVEMENTS,
|
||||
SHOT_TYPES,
|
||||
detailedDescriptionFromShots,
|
||||
formatCutTime,
|
||||
linePart,
|
||||
newLine,
|
||||
newShot,
|
||||
newTextPart,
|
||||
parseCutTime,
|
||||
parseDetailedDescription,
|
||||
parseSubjectLabels,
|
||||
resolveShotStarts,
|
||||
retentionAnalysisFromShots,
|
||||
shotLines,
|
||||
withFreshIds,
|
||||
type CameraMovement,
|
||||
type DialogueLine,
|
||||
type LinePart,
|
||||
type ShotPart,
|
||||
type Shot,
|
||||
type ShotType,
|
||||
} from '@/lib/h3Shots';
|
||||
import { useUndoableState } from '@/hooks/useUndoableState';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
export interface H3ShotListEditorProps {
|
||||
open: boolean;
|
||||
/** Raw subject_definitions text -- parsed for <Subject N> labels to track per shot. */
|
||||
subjectDefinitions: string;
|
||||
initialDetailedDescription: string;
|
||||
initialRetentionAnalysis: string;
|
||||
onApply: (detailedDescription: string, retentionAnalysis: string) => void;
|
||||
onClose: () => void;
|
||||
/** Where the shots get written, for the description text. */
|
||||
targetLabel?: string;
|
||||
/** False when there is no retention analysis to keep in sync (base-format and plain prompts). */
|
||||
withRetention?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structured camera-direction editor for MiniMax-H3: an ordered shot list
|
||||
* (type, movement, which subjects appear, action, optional dialogue) that
|
||||
* writes into detailed_description's [Shot N] blocks and keeps
|
||||
* retention_analysis's "appears in [...]" lines in sync automatically,
|
||||
* instead of hand-maintaining both fields' free text in agreement.
|
||||
*/
|
||||
/** Cut-in time field: blank means "automatic" (shown as the placeholder); Enter or blur commits. */
|
||||
function CutTimeInput({
|
||||
value,
|
||||
auto,
|
||||
disabled,
|
||||
onCommit,
|
||||
}: {
|
||||
value: number | null;
|
||||
auto: number;
|
||||
disabled?: boolean;
|
||||
onCommit: (seconds: number | null) => void;
|
||||
}) {
|
||||
const show = (v: number | null) => (v == null ? '' : formatCutTime(v));
|
||||
const [text, setText] = React.useState(show(value));
|
||||
React.useEffect(() => setText(show(value)), [value]);
|
||||
|
||||
function commit() {
|
||||
const trimmed = text.trim();
|
||||
if (!trimmed) return onCommit(null);
|
||||
const parsed = parseCutTime(trimmed);
|
||||
if (parsed == null) return setText(show(value));
|
||||
onCommit(parsed);
|
||||
}
|
||||
|
||||
return (
|
||||
<Input
|
||||
value={text}
|
||||
onChange={(e) => setText(e.target.value)}
|
||||
onBlur={commit}
|
||||
onKeyDown={(e) => {
|
||||
if (e.key === 'Enter') {
|
||||
e.preventDefault();
|
||||
commit();
|
||||
}
|
||||
}}
|
||||
placeholder={disabled ? 'Opening shot' : formatCutTime(auto)}
|
||||
disabled={disabled}
|
||||
inputMode="decimal"
|
||||
className="font-mono"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
interface EditorState {
|
||||
shots: Shot[];
|
||||
preamble: string;
|
||||
}
|
||||
|
||||
export function H3ShotListEditor({
|
||||
open,
|
||||
subjectDefinitions,
|
||||
initialDetailedDescription,
|
||||
initialRetentionAnalysis,
|
||||
onApply,
|
||||
onClose,
|
||||
targetLabel = 'Detailed description',
|
||||
withRetention = true,
|
||||
}: H3ShotListEditorProps) {
|
||||
// One history covers everything editable here: the shots (structure, fields and
|
||||
// every line of text) and the scene notes. Typing carries a key so a burst
|
||||
// merges into one step; everything else is a step of its own.
|
||||
const history = useUndoableState<EditorState>({ shots: [], preamble: '' });
|
||||
const { shots, preamble } = history.state;
|
||||
// Returning the same list (or text) hands the history back the same state object,
|
||||
// which it treats as "nothing changed" and does not record.
|
||||
const editShots = (fn: (shots: Shot[]) => Shot[], key: string | null = null) =>
|
||||
history.set((s) => {
|
||||
const next = fn(s.shots);
|
||||
return next === s.shots ? s : { ...s, shots: next };
|
||||
}, key);
|
||||
const setPreamble = (value: string) =>
|
||||
history.set((s) => (s.preamble === value ? s : { ...s, preamble: value }), 'preamble');
|
||||
const [selectedId, setSelectedId] = React.useState<string | null>(null);
|
||||
// The shared tag palette inserts into whichever shot's dialogue was last focused.
|
||||
const [dialogueTargetId, setDialogueTargetId] = React.useState<string | null>(null);
|
||||
const dialogueFields = React.useRef(new Map<string, H3ShotDialogueHandle>());
|
||||
const [importOpen, setImportOpen] = React.useState(false);
|
||||
const subjectLabels = React.useMemo(() => parseSubjectLabels(subjectDefinitions), [subjectDefinitions]);
|
||||
|
||||
// Re-seed from the field's current text each time the editor opens, the
|
||||
// same "seed on the open transition only" guard H3DialogueEditor uses --
|
||||
// otherwise a stray parent re-render while open could clobber in-progress edits.
|
||||
const wasOpenRef = React.useRef(false);
|
||||
React.useEffect(() => {
|
||||
if (open && !wasOpenRef.current) {
|
||||
const { preamble: seededPreamble, shots: seeded } = parseDetailedDescription(
|
||||
initialDetailedDescription,
|
||||
subjectLabels,
|
||||
);
|
||||
history.reset({ preamble: seededPreamble, shots: seeded });
|
||||
setSelectedId(seeded[0]?.id ?? null);
|
||||
setDialogueTargetId(null);
|
||||
}
|
||||
wasOpenRef.current = open;
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||
}, [open]);
|
||||
|
||||
function addShot() {
|
||||
const shot = newShot();
|
||||
editShots((prev) => [...prev, shot]);
|
||||
selectShot(shot.id);
|
||||
}
|
||||
|
||||
// Storyboard panel -> jump to that shot's editor card.
|
||||
function selectShot(id: string) {
|
||||
setSelectedId(id);
|
||||
requestAnimationFrame(() =>
|
||||
document.getElementById(`shot-card-${id}`)?.scrollIntoView({ behavior: 'smooth', block: 'nearest' }),
|
||||
);
|
||||
}
|
||||
|
||||
function removeShot(id: string) {
|
||||
editShots((prev) => prev.filter((s) => s.id !== id));
|
||||
setDialogueTargetId((cur) => (cur === id ? null : cur));
|
||||
}
|
||||
|
||||
function insertTag(tag: H3Tag) {
|
||||
if (dialogueTargetId) dialogueFields.current.get(dialogueTargetId)?.insertTag(tag);
|
||||
}
|
||||
|
||||
function applyPreset(preset: H3Preset) {
|
||||
if (dialogueTargetId) dialogueFields.current.get(dialogueTargetId)?.applyPreset(preset);
|
||||
}
|
||||
|
||||
const partKey = (p: ShotPart) => (p.kind === 'text' ? p.id : p.line.id);
|
||||
|
||||
// An edit that changes nothing returns the same list and so records no undo step
|
||||
// (a blur that re-commits the same value, for instance).
|
||||
function updateBody(shotId: string, fn: (body: ShotPart[]) => ShotPart[], editKey: string | null = null) {
|
||||
editShots((prev) => {
|
||||
let changed = false;
|
||||
const next = prev.map((s) => {
|
||||
if (s.id !== shotId) return s;
|
||||
const body = fn(s.body);
|
||||
if (body === s.body) return s;
|
||||
changed = true;
|
||||
return { ...s, body };
|
||||
});
|
||||
return changed ? next : prev;
|
||||
}, editKey);
|
||||
}
|
||||
|
||||
function addLine(shotId: string) {
|
||||
updateBody(shotId, (body) => {
|
||||
const lastSpeaker = [...body].reverse().find((p): p is LinePart => p.kind === 'line')?.line.speaker;
|
||||
return [...body, linePart(newLine({ speaker: lastSpeaker ?? 'S1' }))];
|
||||
});
|
||||
}
|
||||
|
||||
function addText(shotId: string) {
|
||||
updateBody(shotId, (body) => [...body, newTextPart()]);
|
||||
}
|
||||
|
||||
function removePart(shotId: string, key: string) {
|
||||
updateBody(shotId, (body) => body.filter((p) => partKey(p) !== key));
|
||||
setDialogueTargetId((cur) => (cur === key ? null : cur));
|
||||
}
|
||||
|
||||
function movePart(shotId: string, key: string, dir: -1 | 1) {
|
||||
updateBody(shotId, (body) => {
|
||||
const i = body.findIndex((p) => partKey(p) === key);
|
||||
const j = i + dir;
|
||||
if (i < 0 || j < 0 || j >= body.length) return body;
|
||||
const next = [...body];
|
||||
[next[i], next[j]] = [next[j], next[i]];
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
function updateText(shotId: string, key: string, text: string) {
|
||||
updateBody(
|
||||
shotId,
|
||||
(body) => {
|
||||
const part = body.find((p) => p.kind === 'text' && p.id === key);
|
||||
if (!part || (part.kind === 'text' && part.text === text)) return body;
|
||||
return body.map((p) => (p === part ? { ...part, text } : p));
|
||||
},
|
||||
`text:${key}`,
|
||||
);
|
||||
}
|
||||
|
||||
/** `editKey` groups typing into one undo step; omit it for a discrete edit like a tag insertion. */
|
||||
function updateLine(shotId: string, lineId: string, patch: Partial<DialogueLine>, editKey?: string) {
|
||||
updateBody(
|
||||
shotId,
|
||||
(body) => {
|
||||
const part = body.find((p) => p.kind === 'line' && p.line.id === lineId);
|
||||
if (!part || part.kind !== 'line') return body;
|
||||
const unchanged = (Object.keys(patch) as (keyof DialogueLine)[]).every((k) => part.line[k] === patch[k]);
|
||||
return unchanged ? body : body.map((p) => (p === part ? { ...part, line: { ...part.line, ...patch } } : p));
|
||||
},
|
||||
editKey ?? null,
|
||||
);
|
||||
}
|
||||
|
||||
const undo = () => void history.undo();
|
||||
const redo = () => void history.redo();
|
||||
|
||||
// Undo/redo for the whole list, from anywhere inside it -- including from a textbox,
|
||||
// where the browser's own undo would only know about that one box. Not while the
|
||||
// script import (nested inside this dialog) is open: its box keeps native undo.
|
||||
function handleKeyDown(e: React.KeyboardEvent) {
|
||||
if (importOpen || e.altKey || !(e.metaKey || e.ctrlKey)) return;
|
||||
const k = e.key.toLowerCase();
|
||||
if (k === 'z') {
|
||||
e.preventDefault();
|
||||
if (e.shiftKey) redo();
|
||||
else undo();
|
||||
} else if (k === 'y' && !e.shiftKey) {
|
||||
e.preventDefault();
|
||||
redo();
|
||||
}
|
||||
}
|
||||
|
||||
// An imported clip replaces the list wholesale, as one undoable step.
|
||||
function loadImportedShots(imported: Shot[]) {
|
||||
const next = withFreshIds(imported);
|
||||
editShots(() => next);
|
||||
setSelectedId(next[0]?.id ?? null);
|
||||
setDialogueTargetId(null);
|
||||
}
|
||||
|
||||
const dialogueTarget = (() => {
|
||||
for (let i = 0; i < shots.length; i += 1) {
|
||||
const lines = shotLines(shots[i]);
|
||||
const li = lines.findIndex((l) => l.id === dialogueTargetId);
|
||||
if (li >= 0) return { shot: i + 1, line: li + 1, multi: lines.length > 1 };
|
||||
}
|
||||
return null;
|
||||
})();
|
||||
|
||||
function moveShot(id: string, dir: -1 | 1) {
|
||||
editShots((prev) => {
|
||||
const i = prev.findIndex((s) => s.id === id);
|
||||
const j = i + dir;
|
||||
if (i < 0 || j < 0 || j >= prev.length) return prev;
|
||||
const next = [...prev];
|
||||
[next[i], next[j]] = [next[j], next[i]];
|
||||
return next;
|
||||
});
|
||||
}
|
||||
|
||||
function updateShot(id: string, patch: Partial<Shot>) {
|
||||
editShots((prev) => {
|
||||
const shot = prev.find((s) => s.id === id);
|
||||
const unchanged = shot && (Object.keys(patch) as (keyof Shot)[]).every((k) => shot[k] === patch[k]);
|
||||
return !shot || unchanged ? prev : prev.map((s) => (s === shot ? { ...s, ...patch } : s));
|
||||
});
|
||||
}
|
||||
|
||||
function toggleSubject(id: string, label: string) {
|
||||
editShots((prev) =>
|
||||
prev.map((s) =>
|
||||
s.id === id
|
||||
? {
|
||||
...s,
|
||||
subjectIds: s.subjectIds.includes(label)
|
||||
? s.subjectIds.filter((l) => l !== label)
|
||||
: [...s.subjectIds, label],
|
||||
}
|
||||
: s,
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const starts = React.useMemo(() => resolveShotStarts(shots), [shots]);
|
||||
const previewDetailedDescription = React.useMemo(
|
||||
() => detailedDescriptionFromShots(shots, preamble),
|
||||
[shots, preamble],
|
||||
);
|
||||
const previewRetentionAnalysis = React.useMemo(
|
||||
() => (withRetention ? retentionAnalysisFromShots(shots, subjectLabels, initialRetentionAnalysis) : ''),
|
||||
[withRetention, shots, subjectLabels, initialRetentionAnalysis],
|
||||
);
|
||||
|
||||
function handleApply() {
|
||||
onApply(previewDetailedDescription, previewRetentionAnalysis);
|
||||
onClose();
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={(o) => { if (!o) onClose(); }}>
|
||||
<DialogContent
|
||||
onKeyDown={handleKeyDown}
|
||||
className="flex h-[92vh] max-h-[920px] w-[96vw] max-w-[1100px] flex-col gap-0 overflow-hidden p-0"
|
||||
>
|
||||
<DialogHeader className="border-b border-border px-5 py-4">
|
||||
<DialogTitle>Shot list</DialogTitle>
|
||||
<DialogDescription>
|
||||
Build the scene shot by shot. Applying writes [Shot N] blocks into {targetLabel}
|
||||
{withRetention ? ' and keeps Retention analysis\u2019s \u201cappears in\u201d lines in sync' : ''}.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="flex shrink-0 flex-col gap-2 border-b border-border bg-secondary/10 px-4 py-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-semibold text-foreground">Storyboard</span>
|
||||
<span className="flex items-center gap-1.5">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={undo}
|
||||
disabled={!history.canUndo}
|
||||
title="Undo (Ctrl/Cmd+Z)"
|
||||
>
|
||||
↶ Undo
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={redo}
|
||||
disabled={!history.canRedo}
|
||||
title="Redo (Ctrl/Cmd+Shift+Z)"
|
||||
>
|
||||
↷ Redo
|
||||
</Button>
|
||||
<Button type="button" variant="outline" size="sm" onClick={() => setImportOpen(true)}>
|
||||
Import script…
|
||||
</Button>
|
||||
</span>
|
||||
</div>
|
||||
<H3ShotStoryboard shots={shots} selectedId={selectedId} onSelect={selectShot} onAdd={addShot} />
|
||||
</div>
|
||||
|
||||
<div className="flex min-h-0 flex-1 flex-col md:flex-row">
|
||||
<div className="flex min-h-0 min-w-0 flex-1 flex-col gap-4 overflow-y-auto p-4">
|
||||
<label className="flex flex-col gap-1 text-xs text-muted-foreground">
|
||||
Scene notes (optional, written before Shot 1 -- style, lighting, anything true of the whole clip)
|
||||
<Textarea
|
||||
value={preamble}
|
||||
onChange={(e) => setPreamble(e.target.value)}
|
||||
rows={2}
|
||||
placeholder="e.g. Live-action, cinematic, shallow depth of field."
|
||||
/>
|
||||
</label>
|
||||
|
||||
{shots.length === 0 && (
|
||||
<p className="rounded-xl border border-dashed border-border p-4 text-sm text-muted-foreground">
|
||||
No shots yet.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{shots.map((shot, i) => (
|
||||
<div
|
||||
key={shot.id}
|
||||
id={`shot-card-${shot.id}`}
|
||||
onFocusCapture={() => setSelectedId(shot.id)}
|
||||
className={cn(
|
||||
'flex flex-col gap-3 rounded-xl border bg-secondary/20 p-3',
|
||||
shot.id === selectedId ? 'border-accent-blue' : 'border-border',
|
||||
)}
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-sm font-semibold text-foreground">
|
||||
Shot {i + 1}
|
||||
{i > 0 && (
|
||||
<span className="ml-2 font-mono text-xs font-normal text-muted-foreground">
|
||||
cuts in at {formatCutTime(starts[i])}
|
||||
</span>
|
||||
)}
|
||||
</span>
|
||||
<div className="flex items-center gap-1">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => moveShot(shot.id, -1)}
|
||||
disabled={i === 0}
|
||||
title="Move up"
|
||||
>
|
||||
↑
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => moveShot(shot.id, 1)}
|
||||
disabled={i === shots.length - 1}
|
||||
title="Move down"
|
||||
>
|
||||
↓
|
||||
</Button>
|
||||
<Button type="button" variant="outline" size="sm" onClick={() => removeShot(shot.id)}>
|
||||
Remove
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div className="grid grid-cols-1 gap-2 sm:grid-cols-3">
|
||||
<label className="flex flex-col gap-1 text-xs text-muted-foreground">
|
||||
Shot type
|
||||
<NativeSelect
|
||||
value={shot.type}
|
||||
onChange={(e) => updateShot(shot.id, { type: e.target.value as ShotType, freeLead: false })}
|
||||
>
|
||||
{SHOT_TYPES.map((t) => (
|
||||
<option key={t.id} value={t.id}>{t.label}</option>
|
||||
))}
|
||||
</NativeSelect>
|
||||
</label>
|
||||
<label className="flex flex-col gap-1 text-xs text-muted-foreground">
|
||||
Camera movement
|
||||
<NativeSelect
|
||||
value={shot.movement}
|
||||
onChange={(e) => updateShot(shot.id, { movement: e.target.value as CameraMovement, freeLead: false })}
|
||||
>
|
||||
{CAMERA_MOVEMENTS.map((m) => (
|
||||
<option key={m.id} value={m.id}>{m.label}</option>
|
||||
))}
|
||||
</NativeSelect>
|
||||
</label>
|
||||
<label className="flex flex-col gap-1 text-xs text-muted-foreground">
|
||||
Cut time (blank = automatic)
|
||||
<CutTimeInput
|
||||
value={shot.startSeconds}
|
||||
auto={starts[i]}
|
||||
disabled={i === 0}
|
||||
onCommit={(startSeconds) => updateShot(shot.id, { startSeconds })}
|
||||
/>
|
||||
</label>
|
||||
</div>
|
||||
|
||||
{shot.freeLead && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
This shot keeps its original wording, so type and movement aren't written. Changing either
|
||||
adds a generated lead-in in front of the text below.
|
||||
</p>
|
||||
)}
|
||||
|
||||
{subjectLabels.length > 0 && (
|
||||
<div className="flex flex-col gap-1">
|
||||
<span className="text-xs text-muted-foreground">Subjects in this shot</span>
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{subjectLabels.map((label) => {
|
||||
const active = shot.subjectIds.includes(label);
|
||||
return (
|
||||
<button
|
||||
key={label}
|
||||
type="button"
|
||||
aria-pressed={active}
|
||||
onClick={() => toggleSubject(shot.id, label)}
|
||||
className={cn(
|
||||
'rounded-full border px-2 py-0.5 font-mono text-[11px] transition-colors',
|
||||
active
|
||||
? 'border-accent-blue bg-accent-blue/15 text-accent-blue'
|
||||
: 'border-input bg-card text-muted-foreground hover:bg-accent',
|
||||
)}
|
||||
>
|
||||
{label}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex flex-col gap-2">
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Description and dialogue, in the order they play. Dialogue is wrapped in <d> on apply.
|
||||
</span>
|
||||
{(() => {
|
||||
let lineNo = 0;
|
||||
let textNo = 0;
|
||||
return shot.body.map((part, bi) => {
|
||||
const key = partKey(part);
|
||||
const noun = part.kind === 'line' ? `line ${(lineNo += 1)}` : `text ${(textNo += 1)}`;
|
||||
const controls = (
|
||||
<div className="mt-6 flex shrink-0 gap-1">
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => movePart(shot.id, key, -1)}
|
||||
disabled={bi === 0}
|
||||
aria-label={`Move ${noun} up`}
|
||||
title="Move up"
|
||||
>
|
||||
↑
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => movePart(shot.id, key, 1)}
|
||||
disabled={bi === shot.body.length - 1}
|
||||
aria-label={`Move ${noun} down`}
|
||||
title="Move down"
|
||||
>
|
||||
↓
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
onClick={() => removePart(shot.id, key)}
|
||||
aria-label={`Remove ${noun}`}
|
||||
title="Remove"
|
||||
>
|
||||
×
|
||||
</Button>
|
||||
</div>
|
||||
);
|
||||
|
||||
if (part.kind === 'text') {
|
||||
return (
|
||||
<div key={key} className="flex items-start gap-2">
|
||||
<label className="flex min-w-0 flex-1 flex-col gap-1 text-xs text-muted-foreground">
|
||||
{textNo === 1 ? 'Description (composition, action, environment)' : `Text ${textNo}`}
|
||||
<Textarea
|
||||
value={part.text}
|
||||
onChange={(e) => updateText(shot.id, key, e.target.value)}
|
||||
rows={2}
|
||||
placeholder="What's happening in this shot"
|
||||
/>
|
||||
</label>
|
||||
{controls}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const line = part.line;
|
||||
return (
|
||||
<div key={key} className="flex items-start gap-2">
|
||||
<Input
|
||||
value={line.speaker}
|
||||
onChange={(e) => updateLine(shot.id, line.id, { speaker: e.target.value }, `speaker:${line.id}`)}
|
||||
placeholder="S1"
|
||||
aria-label={`Line ${lineNo} speaker`}
|
||||
className="mt-7 w-16 shrink-0 font-mono"
|
||||
/>
|
||||
<div className="min-w-0 flex-1">
|
||||
<H3ShotDialogueField
|
||||
ref={(handle) => {
|
||||
if (handle) dialogueFields.current.set(line.id, handle);
|
||||
else dialogueFields.current.delete(line.id);
|
||||
}}
|
||||
value={line.text}
|
||||
onChange={(text) => updateLine(shot.id, line.id, { text }, `line:${line.id}`)}
|
||||
onInsert={(text) => updateLine(shot.id, line.id, { text })}
|
||||
onFocus={() => setDialogueTargetId(line.id)}
|
||||
active={line.id === dialogueTargetId}
|
||||
label={`Line ${lineNo}`}
|
||||
/>
|
||||
</div>
|
||||
{controls}
|
||||
</div>
|
||||
);
|
||||
});
|
||||
})()}
|
||||
<div className="flex gap-2">
|
||||
<Button type="button" variant="outline" size="sm" onClick={() => addText(shot.id)}>
|
||||
+ Add text
|
||||
</Button>
|
||||
<Button type="button" variant="outline" size="sm" onClick={() => addLine(shot.id)}>
|
||||
+ Add line
|
||||
</Button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
|
||||
<Button type="button" variant="outline" onClick={addShot} className="self-start">
|
||||
+ Add shot
|
||||
</Button>
|
||||
|
||||
<div className="flex flex-col gap-2 rounded-xl border border-border bg-card p-3">
|
||||
<span className="text-xs font-semibold text-foreground">Preview</span>
|
||||
<div className="flex flex-col gap-1">
|
||||
<span className="text-[11px] font-medium uppercase tracking-wide text-muted-foreground">
|
||||
{targetLabel}
|
||||
</span>
|
||||
<pre className="whitespace-pre-wrap rounded-lg bg-secondary/30 p-2 font-mono text-[11px] text-foreground">
|
||||
{previewDetailedDescription || '—'}
|
||||
</pre>
|
||||
</div>
|
||||
{withRetention && (
|
||||
<div className="flex flex-col gap-1">
|
||||
<span className="text-[11px] font-medium uppercase tracking-wide text-muted-foreground">
|
||||
Retention analysis
|
||||
</span>
|
||||
<pre className="whitespace-pre-wrap rounded-lg bg-secondary/30 p-2 font-mono text-[11px] text-foreground">
|
||||
{previewRetentionAnalysis || '—'}
|
||||
</pre>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<aside className="flex max-h-56 shrink-0 flex-col border-t border-border md:max-h-none md:w-64 md:border-l md:border-t-0">
|
||||
<p className="border-b border-border px-3 py-2 text-xs text-muted-foreground">
|
||||
{dialogueTarget
|
||||
? `Tags insert into Shot ${dialogueTarget.shot}${dialogueTarget.multi ? `, line ${dialogueTarget.line}` : ''} dialogue`
|
||||
: 'Click into a shot\'s dialogue to insert tags'}
|
||||
</p>
|
||||
<H3TagPalette
|
||||
onInsertTag={insertTag}
|
||||
onApplyPreset={applyPreset}
|
||||
disabled={!dialogueTarget}
|
||||
className="overflow-y-auto p-3"
|
||||
/>
|
||||
</aside>
|
||||
</div>
|
||||
|
||||
<DialogFooter className="border-t border-border px-5 py-3">
|
||||
<Button type="button" variant="outline" onClick={onClose}>Cancel</Button>
|
||||
<Button type="button" onClick={handleApply}>Apply</Button>
|
||||
</DialogFooter>
|
||||
|
||||
{/* Nested so Radix treats it as part of this dialog (see CreateJobModal). */}
|
||||
<H3ScriptImportDialog
|
||||
open={importOpen}
|
||||
subjectLabels={subjectLabels}
|
||||
existingShotCount={shots.length}
|
||||
onLoad={loadImportedShots}
|
||||
onClose={() => setImportOpen(false)}
|
||||
/>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,214 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import {
|
||||
CAMERA_MOVEMENTS,
|
||||
SHOT_TYPES,
|
||||
shotLines,
|
||||
shotProse,
|
||||
type CameraMovement,
|
||||
type Shot,
|
||||
type ShotType,
|
||||
} from '@/lib/h3Shots';
|
||||
|
||||
const TYPE_LABEL = Object.fromEntries(SHOT_TYPES.map((t) => [t.id, t.label])) as Record<ShotType, string>;
|
||||
const MOVEMENT_LABEL = Object.fromEntries(CAMERA_MOVEMENTS.map((m) => [m.id, m.label])) as Record<
|
||||
CameraMovement,
|
||||
string
|
||||
>;
|
||||
|
||||
// Schematic framing per shot type: `scale` sizes the figures and `baseY` is
|
||||
// where their shoulders bottom out in the 160x90 frame, so a wide shot reads
|
||||
// as small distant figures and a close-up as a head filling the frame.
|
||||
const FRAMING: Partial<Record<ShotType, { scale: number; baseY: number }>> = {
|
||||
establishing: { scale: 0.45, baseY: 72 },
|
||||
wide: { scale: 0.8, baseY: 78 },
|
||||
medium: { scale: 1.5, baseY: 92 },
|
||||
'close-up': { scale: 2.6, baseY: 118 },
|
||||
};
|
||||
|
||||
function figureXs(count: number): number[] {
|
||||
if (count <= 1) return [80];
|
||||
if (count === 2) return [55, 105];
|
||||
return [38, 80, 122];
|
||||
}
|
||||
|
||||
function Figure({ x, baseY, scale, opacity }: { x: number; baseY: number; scale: number; opacity: number }) {
|
||||
return (
|
||||
<g fill="currentColor" opacity={opacity}>
|
||||
<circle cx={x} cy={baseY - 30 * scale} r={8 * scale} />
|
||||
<ellipse cx={x} cy={baseY - 12 * scale} rx={14 * scale} ry={12 * scale} />
|
||||
</g>
|
||||
);
|
||||
}
|
||||
|
||||
export function ShotFrame({ shot }: { shot: Shot }) {
|
||||
const count = Math.min(Math.max(shot.subjectIds.length, 1), 3);
|
||||
const empty = shot.subjectIds.length === 0;
|
||||
const opacity = empty ? 0.25 : 0.6;
|
||||
const framing = FRAMING[shot.type];
|
||||
|
||||
let scene: React.ReactNode;
|
||||
if (shot.type === 'extreme-close-up') {
|
||||
scene = (
|
||||
<g fill="currentColor" opacity={opacity}>
|
||||
<circle cx={80} cy={52} r={62} />
|
||||
<ellipse cx={60} cy={42} rx={7} ry={4} className="fill-background" />
|
||||
<ellipse cx={100} cy={42} rx={7} ry={4} className="fill-background" />
|
||||
</g>
|
||||
);
|
||||
} else if (shot.type === 'over-the-shoulder') {
|
||||
scene = (
|
||||
<>
|
||||
<Figure x={22} baseY={135} scale={3} opacity={0.85} />
|
||||
<Figure x={116} baseY={80} scale={1} opacity={opacity} />
|
||||
</>
|
||||
);
|
||||
} else if (shot.type === 'pov') {
|
||||
scene = (
|
||||
<>
|
||||
<Figure x={80} baseY={72} scale={0.9} opacity={opacity} />
|
||||
<path
|
||||
d="M0 0H160V90H0Z M80 45m-52 0a52 40 0 1 0 104 0a52 40 0 1 0 -104 0"
|
||||
fill="currentColor"
|
||||
fillRule="evenodd"
|
||||
opacity={0.3}
|
||||
/>
|
||||
<path d="M74 45H86M80 39V51" stroke="currentColor" strokeWidth={1.5} opacity={0.6} />
|
||||
</>
|
||||
);
|
||||
} else if (shot.type === 'insert') {
|
||||
scene = (
|
||||
<g opacity={0.6} fill="none" stroke="currentColor" strokeWidth={2.5}>
|
||||
<rect x={52} y={16} width={56} height={58} rx={8} />
|
||||
<path d="M64 34H96M64 46H96M64 58H84" strokeLinecap="round" />
|
||||
</g>
|
||||
);
|
||||
} else if (framing) {
|
||||
scene = (
|
||||
<>
|
||||
{shot.type === 'establishing' && (
|
||||
<path d="M0 66H160" stroke="currentColor" strokeWidth={1} opacity={0.35} />
|
||||
)}
|
||||
{figureXs(count).map((x, i) => (
|
||||
<Figure
|
||||
key={x}
|
||||
x={shot.type === 'close-up' ? 80 : x}
|
||||
baseY={framing.baseY}
|
||||
scale={framing.scale}
|
||||
opacity={opacity * (i === 0 ? 1 : 0.75)}
|
||||
/>
|
||||
))}
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
return (
|
||||
<svg viewBox="0 0 160 90" className="block h-full w-full text-foreground" aria-hidden>
|
||||
{scene}
|
||||
<MovementOverlay movement={shot.movement} />
|
||||
</svg>
|
||||
);
|
||||
}
|
||||
|
||||
function MovementOverlay({ movement }: { movement: CameraMovement }) {
|
||||
const g = (children: React.ReactNode) => (
|
||||
<g
|
||||
className="text-accent-blue"
|
||||
stroke="currentColor"
|
||||
fill="none"
|
||||
strokeWidth={2.5}
|
||||
strokeLinecap="round"
|
||||
strokeLinejoin="round"
|
||||
>
|
||||
{children}
|
||||
</g>
|
||||
);
|
||||
switch (movement) {
|
||||
case 'pan':
|
||||
return g(<path d="M28 80H132M28 80l6-4M28 80l6 4M132 80l-6-4M132 80l-6 4" />);
|
||||
case 'tilt':
|
||||
return g(<path d="M146 18V72M146 18l-4 6M146 18l4 6M146 72l-4-6M146 72l4-6" />);
|
||||
case 'dolly':
|
||||
return g(<path d="M10 86L54 56M150 86L106 56M10 86H150" strokeDasharray="4 4" />);
|
||||
case 'zoom':
|
||||
return g(<path d="M16 26V14H28M132 14H144V26M16 64V76H28M132 76H144V64" />);
|
||||
case 'handheld':
|
||||
return g(
|
||||
<rect x={7} y={7} width={146} height={76} rx={4} strokeDasharray="3 4" transform="rotate(-1.6 80 45)" />,
|
||||
);
|
||||
case 'tracking':
|
||||
return g(
|
||||
<>
|
||||
<path d="M22 82H134M134 82l-6-4M134 82l-6 4" />
|
||||
<circle cx={30} cy={82} r={3} fill="currentColor" />
|
||||
</>,
|
||||
);
|
||||
case 'crane':
|
||||
return g(<path d="M24 80L134 20M134 20h-9M134 20l-3 8" />);
|
||||
default:
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function subjectShortLabel(label: string): string {
|
||||
const m = label.match(/(\d+)/);
|
||||
return m ? `Subj ${m[1]}` : label;
|
||||
}
|
||||
|
||||
export interface H3ShotPanelProps {
|
||||
shot: Shot;
|
||||
/** the number shown in the badge */
|
||||
number: number | string;
|
||||
/** time printed on the frame, e.g. "00:03.500" */
|
||||
timeLabel: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The contents of one storyboard panel: badge, shot type and camera move, a
|
||||
* schematic frame, who is in it, what is said, and the shot's prose. Callers
|
||||
* supply the surrounding card (a button in the editor, plain in the scene view).
|
||||
*/
|
||||
export function H3ShotPanel({ shot, number, timeLabel }: H3ShotPanelProps) {
|
||||
const spoken = shotLines(shot).filter((l) => l.text.trim());
|
||||
return (
|
||||
<>
|
||||
<div className="flex items-center gap-1.5">
|
||||
<span className="flex size-5 shrink-0 items-center justify-center rounded-full bg-secondary text-[11px] font-semibold text-foreground">
|
||||
{number}
|
||||
</span>
|
||||
<span className="truncate text-xs font-medium text-foreground">{TYPE_LABEL[shot.type]}</span>
|
||||
<span className="ml-auto shrink-0 text-[10px] text-muted-foreground">{MOVEMENT_LABEL[shot.movement]}</span>
|
||||
</div>
|
||||
|
||||
<div className="relative aspect-video overflow-hidden rounded-md border border-border bg-secondary/40">
|
||||
<ShotFrame shot={shot} />
|
||||
<span className="absolute bottom-1 left-1 rounded bg-background/80 px-1 font-mono text-[9px] text-muted-foreground">
|
||||
{timeLabel}
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{shot.subjectIds.length > 0 && (
|
||||
<div className="flex flex-wrap gap-1">
|
||||
{shot.subjectIds.map((label) => (
|
||||
<span
|
||||
key={label}
|
||||
className="rounded-full border border-input bg-secondary/50 px-1.5 py-px font-mono text-[10px] text-muted-foreground"
|
||||
>
|
||||
{subjectShortLabel(label)}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{spoken.length > 0 && (
|
||||
<p className="line-clamp-3 rounded-md bg-accent-blue/10 px-1.5 py-1 text-[11px] italic text-foreground">
|
||||
{spoken.map((l) => `${l.speaker.trim() ? `${l.speaker.trim()}: ` : ''}“${l.text.trim()}”`).join(' ')}
|
||||
</p>
|
||||
)}
|
||||
|
||||
<p className="line-clamp-2 text-[11px] text-muted-foreground">{shotProse(shot) || 'No description yet'}</p>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { H3ShotStoryboard } from './H3ShotStoryboard';
|
||||
import { CAMERA_MOVEMENTS, SHOT_TYPES, makeShot, newLine, type Shot, type ShotInit } from '@/lib/h3Shots';
|
||||
|
||||
function shot(init: ShotInit): Shot {
|
||||
return makeShot(init);
|
||||
}
|
||||
|
||||
describe('H3ShotStoryboard', () => {
|
||||
it('lays shots out in order with type, movement, subjects, and dialogue', () => {
|
||||
const shots = [
|
||||
shot({ type: 'establishing', description: 'Wide rooftop.', subjectIds: ['<Subject 1>', '<Subject 2>'] }),
|
||||
shot({ type: 'close-up', movement: 'handheld', lines: [newLine({ speaker: 'S2', text: 'I had no choice.' }), newLine({ speaker: 'S1', text: 'Then go.' })], subjectIds: ['<Subject 2>'] }),
|
||||
];
|
||||
render(<H3ShotStoryboard shots={shots} selectedId={shots[1].id} onSelect={() => {}} onAdd={() => {}} />);
|
||||
|
||||
expect(screen.getByText('Establishing')).toBeInTheDocument();
|
||||
expect(screen.getByText('Close-up')).toBeInTheDocument();
|
||||
expect(screen.getByText('Handheld')).toBeInTheDocument();
|
||||
expect(screen.getByText(/S2: “I had no choice.” S1: “Then go.”/)).toBeInTheDocument();
|
||||
expect(screen.getAllByText('Subj 2')).toHaveLength(2);
|
||||
|
||||
const panels = screen.getAllByRole('button').filter((b) => b.hasAttribute('aria-current') || b.textContent?.match(/^\d/));
|
||||
expect(panels[0]).toHaveTextContent(/^1Establishing/);
|
||||
expect(screen.getByRole('button', { name: /Close-up/ })).toHaveAttribute('aria-current', 'true');
|
||||
});
|
||||
|
||||
it('selects a shot on click and adds via the trailing tile', async () => {
|
||||
const user = userEvent.setup();
|
||||
const onSelect = vi.fn();
|
||||
const onAdd = vi.fn();
|
||||
const shots = [shot({ description: 'First.' }), shot({ description: 'Second.' })];
|
||||
render(<H3ShotStoryboard shots={shots} selectedId={null} onSelect={onSelect} onAdd={onAdd} />);
|
||||
|
||||
await user.click(screen.getByRole('button', { name: /Second\./ }));
|
||||
expect(onSelect).toHaveBeenCalledWith(shots[1].id);
|
||||
await user.click(screen.getByRole('button', { name: /Add shot/ }));
|
||||
expect(onAdd).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('renders every shot type and camera movement without crashing', () => {
|
||||
const shots = SHOT_TYPES.flatMap((t) =>
|
||||
CAMERA_MOVEMENTS.map((m) => shot({ type: t.id, movement: m.id, subjectIds: ['<Subject 1>'] })),
|
||||
);
|
||||
const { container } = render(<H3ShotStoryboard shots={shots} selectedId={null} onSelect={() => {}} onAdd={() => {}} />);
|
||||
expect(container.querySelectorAll('svg')).toHaveLength(SHOT_TYPES.length * CAMERA_MOVEMENTS.length);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { H3ShotPanel } from '@/components/jobs/H3ShotPanel';
|
||||
import { formatCutTime, resolveShotStarts, type Shot } from '@/lib/h3Shots';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
export interface H3ShotStoryboardProps {
|
||||
shots: Shot[];
|
||||
selectedId: string | null;
|
||||
onSelect: (id: string) => void;
|
||||
onAdd: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* The shot list as a storyboard: one panel per shot, left to right in
|
||||
* playback order. Panels are schematic (no generated imagery exists yet) --
|
||||
* subject size reflects shot type and an overlay marks camera movement --
|
||||
* so the sequence, who's in frame, and where the dialogue falls read at a
|
||||
* glance without scrolling the per-shot editors.
|
||||
*/
|
||||
export function H3ShotStoryboard({ shots, selectedId, onSelect, onAdd }: H3ShotStoryboardProps) {
|
||||
const starts = React.useMemo(() => resolveShotStarts(shots), [shots]);
|
||||
return (
|
||||
<div className="flex items-stretch gap-2 overflow-x-auto pb-1" aria-label="Storyboard">
|
||||
{shots.map((shot, i) => {
|
||||
const selected = shot.id === selectedId;
|
||||
return (
|
||||
<React.Fragment key={shot.id}>
|
||||
{i > 0 && (
|
||||
<span className="flex shrink-0 items-center text-muted-foreground/60" aria-hidden>
|
||||
›
|
||||
</span>
|
||||
)}
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => onSelect(shot.id)}
|
||||
aria-current={selected ? 'true' : undefined}
|
||||
className={cn(
|
||||
'flex w-52 shrink-0 flex-col gap-1.5 rounded-xl border bg-card p-2 text-left transition-colors hover:bg-accent/40',
|
||||
selected ? 'border-accent-blue ring-2 ring-accent-blue/50' : 'border-border',
|
||||
)}
|
||||
>
|
||||
<H3ShotPanel shot={shot} number={i + 1} timeLabel={formatCutTime(starts[i])} />
|
||||
</button>
|
||||
</React.Fragment>
|
||||
);
|
||||
})}
|
||||
|
||||
<button
|
||||
type="button"
|
||||
onClick={onAdd}
|
||||
className="flex min-h-32 w-28 shrink-0 flex-col items-center justify-center gap-1 rounded-xl border border-dashed border-border text-xs text-muted-foreground transition-colors hover:bg-accent/40 hover:text-foreground"
|
||||
>
|
||||
<span className="text-lg leading-none">+</span>
|
||||
Add shot
|
||||
</button>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { H3_TAG_CATEGORIES, type H3Tag } from '@/lib/h3Tags';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
export interface H3TagConsoleProps {
|
||||
/** Called with the tag to insert into whichever field is currently focused. */
|
||||
onInsert: (tag: H3Tag) => void;
|
||||
disabled?: boolean;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A microexpression "console" for MiniMax-H3 speech tags: click a tag to
|
||||
* insert it at the cursor of whichever prompt field was last focused. Kept
|
||||
* as plain buttons (not a menu) so every tag stays one click away.
|
||||
*/
|
||||
export function H3TagConsole({ onInsert, disabled, className }: H3TagConsoleProps) {
|
||||
return (
|
||||
<div
|
||||
className={cn(
|
||||
'flex flex-col gap-1.5 rounded-xl border border-input bg-secondary/40 p-2',
|
||||
className,
|
||||
)}
|
||||
>
|
||||
<span className="px-0.5 text-[11px] font-medium uppercase tracking-wide text-muted-foreground">
|
||||
Speech tags — click to insert at the cursor
|
||||
</span>
|
||||
<div className="flex flex-col gap-1.5">
|
||||
{H3_TAG_CATEGORIES.map((category) => (
|
||||
<div key={category.id} className="flex flex-wrap items-center gap-1">
|
||||
<span className="mr-0.5 w-full shrink-0 text-[10px] text-muted-foreground/80 sm:w-auto">
|
||||
{category.label}
|
||||
</span>
|
||||
{category.tags.map((tag) => (
|
||||
<button
|
||||
key={tag.id}
|
||||
type="button"
|
||||
title={`${tag.description} · e.g. "${tag.example}"`}
|
||||
// Keep the target textarea focused (and its selection intact)
|
||||
// so the click lands where the user was typing, not nowhere.
|
||||
onMouseDown={(e) => e.preventDefault()}
|
||||
onClick={() => onInsert(tag)}
|
||||
disabled={disabled}
|
||||
className="rounded-full border border-input bg-card px-2 py-0.5 font-mono text-[11px] text-foreground transition-colors hover:bg-accent disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
{tag.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { H3_PRESETS, H3_TAG_CATEGORIES, type H3Preset, type H3Tag } from '@/lib/h3Tags';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
// Reuses the app's existing status hues as a purely visual category key --
|
||||
// no status meaning implied, just four distinguishable accents that already
|
||||
// work in both themes.
|
||||
export const CATEGORY_STYLE: Record<string, { chip: string; dot: string }> = {
|
||||
pace: { chip: 'border-blue-400/35 bg-blue-500/10 text-blue-900 dark:text-blue-100', dot: 'bg-blue-500' },
|
||||
delivery: { chip: 'border-amber-400/35 bg-amber-500/10 text-amber-900 dark:text-amber-100', dot: 'bg-amber-500' },
|
||||
vocal: { chip: 'border-emerald-400/35 bg-emerald-500/10 text-emerald-900 dark:text-emerald-100', dot: 'bg-emerald-500' },
|
||||
body: { chip: 'border-rose-400/35 bg-rose-500/10 text-rose-900 dark:text-rose-100', dot: 'bg-rose-500' },
|
||||
};
|
||||
|
||||
export interface H3TagPaletteProps {
|
||||
onInsertTag: (tag: H3Tag) => void;
|
||||
onApplyPreset: (preset: H3Preset) => void;
|
||||
disabled?: boolean;
|
||||
className?: string;
|
||||
}
|
||||
|
||||
/** Full speech-tag palette + emotion presets. Buttons keep the target textarea focused so the click lands at its cursor. */
|
||||
export function H3TagPalette({ onInsertTag, onApplyPreset, disabled, className }: H3TagPaletteProps) {
|
||||
return (
|
||||
<div className={className}>
|
||||
{H3_TAG_CATEGORIES.map((cat) => {
|
||||
const style = CATEGORY_STYLE[cat.id] ?? CATEGORY_STYLE.body;
|
||||
return (
|
||||
<div key={cat.id} className="mb-4">
|
||||
<div className="mb-1.5 flex items-center gap-2 px-0.5">
|
||||
<span className={cn('size-2 rounded-full', style.dot)} />
|
||||
<span className="text-xs font-semibold text-foreground">{cat.label}</span>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1">
|
||||
{cat.tags.map((tag) => (
|
||||
<button
|
||||
key={tag.id}
|
||||
type="button"
|
||||
title={tag.example}
|
||||
disabled={disabled}
|
||||
onMouseDown={(e) => e.preventDefault()}
|
||||
onClick={() => onInsertTag(tag)}
|
||||
className="flex items-baseline justify-between gap-2 rounded-lg border border-input bg-card px-2.5 py-1.5 text-left transition-colors hover:bg-accent disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
<span className="font-mono text-[11.5px] text-foreground">{tag.label}</span>
|
||||
<span className="truncate text-[10.5px] text-muted-foreground">{tag.description}</span>
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<div className="mb-1.5 mt-5 px-0.5 text-xs font-semibold text-foreground">Emotion presets</div>
|
||||
<div className="flex flex-wrap gap-1.5">
|
||||
{H3_PRESETS.map((p) => (
|
||||
<button
|
||||
key={p.label}
|
||||
type="button"
|
||||
disabled={disabled}
|
||||
onMouseDown={(e) => e.preventDefault()}
|
||||
onClick={() => onApplyPreset(p)}
|
||||
className="rounded-full border border-input bg-card px-2.5 py-1 text-[11.5px] text-foreground transition-colors hover:bg-accent disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
{p.label}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -5,7 +5,9 @@ import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
import JobCard from '@/components/jobs/JobCard';
|
||||
import {
|
||||
deleteJob,
|
||||
dequeueJob,
|
||||
downloadJobVideo,
|
||||
queueJobs,
|
||||
startJob,
|
||||
stopJob,
|
||||
} from '@/lib/api';
|
||||
@@ -18,6 +20,8 @@ vi.mock('@/lib/api', () => ({
|
||||
stopJob: vi.fn(),
|
||||
deleteJob: vi.fn(),
|
||||
downloadJobVideo: vi.fn(),
|
||||
queueJobs: vi.fn(),
|
||||
dequeueJob: vi.fn(),
|
||||
}));
|
||||
|
||||
vi.mock('@/lib/utils', async (importOriginal) => {
|
||||
@@ -47,6 +51,8 @@ beforeEach(() => {
|
||||
setActiveJobId(null);
|
||||
vi.mocked(startJob).mockResolvedValue({} as Job);
|
||||
vi.mocked(stopJob).mockResolvedValue({} as Job);
|
||||
vi.mocked(queueJobs).mockResolvedValue([]);
|
||||
vi.mocked(dequeueJob).mockResolvedValue({} as Job);
|
||||
vi.mocked(deleteJob).mockResolvedValue(undefined);
|
||||
vi.mocked(downloadJobVideo).mockResolvedValue(new Blob());
|
||||
vi.spyOn(window, 'confirm').mockReturnValue(true);
|
||||
@@ -91,6 +97,46 @@ describe('JobCard', () => {
|
||||
expect(onJobUpdated).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it.each(['pending', 'failed', 'stopped'])('queues a %s job and notifies the parent', async (status) => {
|
||||
const onJobUpdated = vi.fn();
|
||||
render(<JobCard job={makeJob({ status })} onJobUpdated={onJobUpdated} />);
|
||||
await userEvent.click(screen.getByRole('button', { name: 'Queue' }));
|
||||
await waitFor(() => expect(queueJobs).toHaveBeenCalledWith(['job-1']));
|
||||
expect(onJobUpdated).toHaveBeenCalled();
|
||||
expect(startJob).not.toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('reports a job that cannot be queued', async () => {
|
||||
vi.mocked(queueJobs).mockRejectedValue(new Error('Jobs would wait on each other'));
|
||||
render(<JobCard job={makeJob({ status: 'pending' })} />);
|
||||
await userEvent.click(screen.getByRole('button', { name: 'Queue' }));
|
||||
await waitFor(() => expect(window.alert).toHaveBeenCalledWith('Jobs would wait on each other'));
|
||||
});
|
||||
|
||||
it('shows a queued job as queued, and takes it back out of the queue', async () => {
|
||||
const onJobUpdated = vi.fn();
|
||||
render(<JobCard job={makeJob({ status: 'queued' })} onJobUpdated={onJobUpdated} />);
|
||||
|
||||
expect(screen.getByText('queued')).toBeInTheDocument();
|
||||
// nothing to start or queue again; it can't be edited either
|
||||
for (const name of ['Start', 'Restart', 'Queue', 'Stop', 'Edit']) {
|
||||
expect(screen.queryByRole('button', { name })).not.toBeInTheDocument();
|
||||
}
|
||||
expect(screen.getByRole('button', { name: 'View' })).toBeInTheDocument();
|
||||
|
||||
await userEvent.click(screen.getByRole('button', { name: 'Remove from queue' }));
|
||||
await waitFor(() => expect(dequeueJob).toHaveBeenCalledWith('job-1'));
|
||||
expect(onJobUpdated).toHaveBeenCalled();
|
||||
});
|
||||
|
||||
it('does not offer to queue a running or completed job', () => {
|
||||
for (const status of ['running', 'completed']) {
|
||||
const { unmount } = render(<JobCard job={makeJob({ status, started_at: Date.now() })} />);
|
||||
expect(screen.queryByRole('button', { name: 'Queue' })).not.toBeInTheDocument();
|
||||
unmount();
|
||||
}
|
||||
});
|
||||
|
||||
it('stops a running job', async () => {
|
||||
render(<JobCard job={makeJob({ status: 'running', started_at: Date.now() })} />);
|
||||
await userEvent.click(screen.getByRole('button', { name: 'Stop' }));
|
||||
|
||||
@@ -6,11 +6,14 @@ import { Timer } from 'lucide-react';
|
||||
import CreateJobModal from '@/components/jobs/CreateJobModal';
|
||||
import { Badge, type BadgeProps } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { useJobElapsed } from '@/hooks/useJobElapsed';
|
||||
import { useStore } from '@/hooks/useStore';
|
||||
import {
|
||||
deleteJob,
|
||||
duplicateJob,
|
||||
dequeueJob,
|
||||
downloadJobVideo,
|
||||
queueJobs,
|
||||
startJob,
|
||||
stopJob,
|
||||
} from '@/lib/api';
|
||||
@@ -23,34 +26,9 @@ interface JobCardProps {
|
||||
onJobUpdated?: () => void;
|
||||
}
|
||||
|
||||
function formatDuration(seconds: number): string {
|
||||
const roundedSeconds = Math.round(seconds);
|
||||
if (roundedSeconds < 60) return `${roundedSeconds}s`;
|
||||
if (roundedSeconds < 3600) {
|
||||
const mins = Math.floor(roundedSeconds / 60);
|
||||
const secs = roundedSeconds % 60;
|
||||
return secs > 0 ? `${mins}m ${secs}s` : `${mins}m`;
|
||||
}
|
||||
const hours = Math.floor(roundedSeconds / 3600);
|
||||
const mins = Math.floor((roundedSeconds % 3600) / 60);
|
||||
return mins > 0 ? `${hours}h ${mins}m` : `${hours}h`;
|
||||
}
|
||||
|
||||
function computeElapsed(job: Job, currentTime: number): string | null {
|
||||
if (!job.started_at) return null;
|
||||
const endTime =
|
||||
job.status === 'running' ? currentTime : (job.finished_at ?? 0);
|
||||
if (!endTime && job.status !== 'running') return null;
|
||||
const startedAtMs =
|
||||
job.started_at < 1e12 ? job.started_at * 1000 : job.started_at;
|
||||
const endTimeMs = endTime < 1e12 ? endTime * 1000 : endTime;
|
||||
const elapsedSeconds = (endTimeMs - startedAtMs) / 1000;
|
||||
if (elapsedSeconds <= 0) return null;
|
||||
return formatDuration(elapsedSeconds);
|
||||
}
|
||||
|
||||
const BADGE_VARIANTS: Record<string, BadgeProps['variant']> = {
|
||||
pending: 'secondary',
|
||||
queued: 'default',
|
||||
running: 'warning',
|
||||
completed: 'success',
|
||||
ready: 'success',
|
||||
@@ -66,15 +44,7 @@ export default function JobCard({ job, onJobUpdated }: JobCardProps) {
|
||||
const [isLoading, setIsLoading] = React.useState(false);
|
||||
const [isEditing, setIsEditing] = React.useState(false);
|
||||
const [isViewing, setIsViewing] = React.useState(false);
|
||||
const [currentTime, setCurrentTime] = React.useState(() => Date.now());
|
||||
|
||||
const elapsedTime = computeElapsed(job, currentTime);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (job.status !== 'running' || !job.started_at) return;
|
||||
const interval = setInterval(() => setCurrentTime(Date.now()), 1000);
|
||||
return () => clearInterval(interval);
|
||||
}, [job.status, job.started_at]);
|
||||
const elapsedTime = useJobElapsed(job);
|
||||
|
||||
async function handleStart(e: React.MouseEvent) {
|
||||
e.preventDefault();
|
||||
@@ -92,6 +62,36 @@ export default function JobCard({ job, onJobUpdated }: JobCardProps) {
|
||||
}
|
||||
}
|
||||
|
||||
async function handleQueue(e: React.MouseEvent) {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
if (isLoading) return;
|
||||
setIsLoading(true);
|
||||
try {
|
||||
await queueJobs([job.id]);
|
||||
onJobUpdated?.();
|
||||
} catch (err) {
|
||||
alert(err instanceof Error ? err.message : 'Failed to queue job');
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleDequeue(e: React.MouseEvent) {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
if (isLoading) return;
|
||||
setIsLoading(true);
|
||||
try {
|
||||
await dequeueJob(job.id);
|
||||
onJobUpdated?.();
|
||||
} catch (err) {
|
||||
alert(err instanceof Error ? err.message : 'Failed to remove job from the queue');
|
||||
} finally {
|
||||
setIsLoading(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function handleStop(e: React.MouseEvent) {
|
||||
e.preventDefault();
|
||||
e.stopPropagation();
|
||||
@@ -238,7 +238,30 @@ export default function JobCard({ job, onJobUpdated }: JobCardProps) {
|
||||
>
|
||||
Start
|
||||
</Button>
|
||||
) : job.status === 'queued' ? (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={handleDequeue}
|
||||
disabled={isLoading}
|
||||
title="Take this job out of the queue"
|
||||
>
|
||||
Remove from queue
|
||||
</Button>
|
||||
) : null}
|
||||
{(job.status === 'pending' ||
|
||||
job.status === 'failed' ||
|
||||
job.status === 'stopped') && (
|
||||
<Button
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={handleQueue}
|
||||
disabled={isLoading}
|
||||
title="Run this job when the ones queued before it have finished"
|
||||
>
|
||||
Queue
|
||||
</Button>
|
||||
)}
|
||||
{job.status === 'completed' &&
|
||||
job.output_path &&
|
||||
job.job_type === 'inference' && (
|
||||
|
||||
@@ -77,7 +77,7 @@ export default function JobDetailsSidebar({
|
||||
const isRestarting =
|
||||
previousJobId.current === job.id &&
|
||||
wasTerminal &&
|
||||
(job.status === 'pending' || job.status === 'running');
|
||||
(job.status === 'pending' || job.status === 'queued' || job.status === 'running');
|
||||
|
||||
if (previousJobId.current !== job.id || isRestarting) {
|
||||
stateRef.current.text = '';
|
||||
@@ -89,7 +89,10 @@ export default function JobDetailsSidebar({
|
||||
}, [job.id, job.status]);
|
||||
|
||||
React.useEffect(() => {
|
||||
const shouldPoll = job.status === 'running' || job.status === 'pending';
|
||||
const shouldPoll =
|
||||
job.status === 'running' ||
|
||||
job.status === 'pending' ||
|
||||
job.status === 'queued';
|
||||
let pollInterval: ReturnType<typeof setInterval> | null = null;
|
||||
let mounted = true;
|
||||
// Effect-local lock so this job's first fetch is never blocked by a
|
||||
|
||||
@@ -108,7 +108,8 @@ export default function JobQueue({ jobType, jobTypesForList }: JobQueueProps) {
|
||||
|
||||
// Poll every second while any job is running/pending; stop otherwise.
|
||||
const hasActive = jobs.some(
|
||||
(j) => j.status === 'running' || j.status === 'pending',
|
||||
(j) =>
|
||||
j.status === 'running' || j.status === 'pending' || j.status === 'queued',
|
||||
);
|
||||
React.useEffect(() => {
|
||||
if (!hasActive) return;
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { FileAudio, FileVideo, ImageOff } from 'lucide-react';
|
||||
|
||||
import type { MediaType } from '@/lib/api';
|
||||
|
||||
export interface ReferenceThumbnailProps {
|
||||
mediaType: MediaType;
|
||||
/** where the image can be loaded from; null when it cannot be previewed */
|
||||
src: string | null;
|
||||
alt: string;
|
||||
}
|
||||
|
||||
/** A small square preview of a reference: the image itself, or an icon for video, audio, or an unreadable image. */
|
||||
export function ReferenceThumbnail({ mediaType, src, alt }: ReferenceThumbnailProps) {
|
||||
const [failed, setFailed] = React.useState(false);
|
||||
React.useEffect(() => setFailed(false), [src]);
|
||||
|
||||
const box = 'flex size-12 shrink-0 items-center justify-center overflow-hidden rounded border border-border bg-muted';
|
||||
if (mediaType === 'image' && src && !failed) {
|
||||
return (
|
||||
<a href={src} target="_blank" rel="noreferrer" className={box} title="Open full size">
|
||||
{/* eslint-disable-next-line @next/next/no-img-element -- served by the Studio API, not a static asset */}
|
||||
<img src={src} alt={alt} loading="lazy" onError={() => setFailed(true)} className="size-full object-cover" />
|
||||
</a>
|
||||
);
|
||||
}
|
||||
const Icon = mediaType === 'video' ? FileVideo : mediaType === 'audio' ? FileAudio : ImageOff;
|
||||
return (
|
||||
<span className={box} aria-hidden>
|
||||
<Icon className="size-5 text-muted-foreground" />
|
||||
</span>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,158 @@
|
||||
import * as React from 'react';
|
||||
import { flushSync } from 'react-dom';
|
||||
import { render, screen, waitFor, within } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import CreateJobModal from './CreateJobModal';
|
||||
import { createJob, getDatasets, getModels } from '@/lib/api';
|
||||
import { defaultOptionsStore } from '@/stores/defaultOptions';
|
||||
import { DEFAULT_OPTIONS } from '@/lib/defaultOptions';
|
||||
|
||||
vi.mock('sonner', () => ({ toast: { warning: vi.fn(), error: vi.fn(), success: vi.fn() } }));
|
||||
vi.mock('@/lib/api', () => ({
|
||||
createJob: vi.fn(),
|
||||
getModels: vi.fn(),
|
||||
getDatasets: vi.fn(),
|
||||
uploadImage: vi.fn(),
|
||||
getSettings: vi.fn(),
|
||||
updateSettings: vi.fn(),
|
||||
}));
|
||||
|
||||
const H3 = { id: 'minimax-h3/i2v', label: 'MiniMax H3', type: 'i2v' };
|
||||
|
||||
// A browser runs a microtask checkpoint -- so React flushes -- between the
|
||||
// listeners of a single native event; jsdom's synchronous dispatch never does.
|
||||
// Radix decides whether a parent dialog should dismiss from the layer stack at
|
||||
// the moment its listener runs, so without this the bug can't show up here.
|
||||
const wrapped = new WeakMap<object, EventListener>();
|
||||
beforeEach(() => {
|
||||
const realAdd = Document.prototype.addEventListener;
|
||||
const realRemove = Document.prototype.removeEventListener;
|
||||
vi.spyOn(document, 'addEventListener').mockImplementation(function (
|
||||
this: Document,
|
||||
type: string,
|
||||
listener: unknown,
|
||||
options?: unknown,
|
||||
) {
|
||||
if (typeof listener === 'function' && ['keydown', 'pointerdown', 'click'].includes(type)) {
|
||||
const flushing: EventListener = (e) => {
|
||||
(listener as EventListener)(e);
|
||||
flushSync(() => {});
|
||||
};
|
||||
wrapped.set(listener, flushing);
|
||||
return realAdd.call(this, type, flushing, options as AddEventListenerOptions);
|
||||
}
|
||||
return realAdd.call(this, type, listener as EventListener, options as AddEventListenerOptions);
|
||||
} as typeof document.addEventListener);
|
||||
vi.spyOn(document, 'removeEventListener').mockImplementation(function (
|
||||
this: Document,
|
||||
type: string,
|
||||
listener: unknown,
|
||||
options?: unknown,
|
||||
) {
|
||||
return realRemove.call(
|
||||
this,
|
||||
type,
|
||||
(wrapped.get(listener as object) ?? listener) as EventListener,
|
||||
options as EventListenerOptions,
|
||||
);
|
||||
} as typeof document.removeEventListener);
|
||||
|
||||
defaultOptionsStore.set({ options: DEFAULT_OPTIONS });
|
||||
vi.mocked(getModels).mockResolvedValue([H3]);
|
||||
vi.mocked(getDatasets).mockResolvedValue([]);
|
||||
vi.mocked(createJob).mockResolvedValue({ id: 'j' } as never);
|
||||
});
|
||||
|
||||
function Harness() {
|
||||
const [open, setOpen] = React.useState(true);
|
||||
return (
|
||||
<>
|
||||
<div data-testid="state">{open ? 'open' : 'closed'}</div>
|
||||
<CreateJobModal
|
||||
isOpen={open}
|
||||
onClose={() => setOpen(false)}
|
||||
onSuccess={() => {}}
|
||||
jobType="inference"
|
||||
workloadType="i2v"
|
||||
/>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
type User = ReturnType<typeof userEvent.setup>;
|
||||
type Closer = (user: User, dialog: HTMLElement) => Promise<void>;
|
||||
|
||||
async function renderH3Modal(user: User) {
|
||||
render(<Harness />);
|
||||
await screen.findByRole('option', { name: /MiniMax H3/ });
|
||||
await user.selectOptions(screen.getByLabelText('Model'), H3.id);
|
||||
}
|
||||
|
||||
// Radix marks every dialog under the topmost one aria-hidden, which also hides its
|
||||
// accessible name, so the job dialog is identified by its title text instead.
|
||||
const expectParentStillOpen = () => {
|
||||
expect(screen.getByTestId('state')).toHaveTextContent('open');
|
||||
expect(screen.getByText(/New Inference Job/)).toBeInTheDocument();
|
||||
};
|
||||
// [0] is the job dialog's overlay; the last is the topmost dialog's.
|
||||
const topOverlay = () => [...document.querySelectorAll('.bg-black\\/70')].at(-1) as Element;
|
||||
|
||||
const CLOSERS: [string, Closer][] = [
|
||||
['Cancel', (u, d) => u.click(within(d).getByRole('button', { name: 'Cancel' }))],
|
||||
['the X button', (u, d) => u.click(within(d).getByRole('button', { name: 'Close' }))],
|
||||
['Escape', (u) => u.keyboard('{Escape}')],
|
||||
['a click on the overlay', (u) => u.click(topOverlay())],
|
||||
];
|
||||
|
||||
describe('closing a dialog opened from the job form', () => {
|
||||
describe.each([
|
||||
['shot list', 'Open shot list', 'Shot list', [...CLOSERS, ['Apply', (u: User, d: HTMLElement) => u.click(within(d).getByRole('button', { name: 'Apply' }))] as [string, Closer]]],
|
||||
['dialogue editor', 'Open dialogue editor', 'Dialogue editor', [...CLOSERS, ['Apply to field', (u: User, d: HTMLElement) => u.click(within(d).getByRole('button', { name: 'Apply to field' }))] as [string, Closer]]],
|
||||
] as const)('%s', (_label, openButton, title, closers) => {
|
||||
it.each(closers)('closing with %s leaves the job form open', async (_name, close) => {
|
||||
const user = userEvent.setup();
|
||||
await renderH3Modal(user);
|
||||
await user.click(await screen.findByRole('button', { name: openButton }));
|
||||
const child = await screen.findByRole('dialog', { name: title });
|
||||
|
||||
await close(user, child);
|
||||
|
||||
await waitFor(() => expect(screen.queryByRole('dialog', { name: title })).not.toBeInTheDocument());
|
||||
expectParentStillOpen();
|
||||
});
|
||||
});
|
||||
|
||||
it('the job form is still usable afterwards', async () => {
|
||||
const user = userEvent.setup();
|
||||
await renderH3Modal(user);
|
||||
await user.click(await screen.findByRole('button', { name: 'Open shot list' }));
|
||||
const child = await screen.findByRole('dialog', { name: 'Shot list' });
|
||||
await user.click(within(child).getByRole('button', { name: 'Cancel' }));
|
||||
await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Shot list' })).not.toBeInTheDocument());
|
||||
|
||||
await user.type(screen.getByLabelText('Name (optional)'), 'still here');
|
||||
expect(screen.getByLabelText('Name (optional)')).toHaveValue('still here');
|
||||
// ...and the dialog can be opened again
|
||||
await user.click(screen.getByRole('button', { name: 'Open shot list' }));
|
||||
expect(await screen.findByRole('dialog', { name: 'Shot list' })).toBeInTheDocument();
|
||||
});
|
||||
|
||||
describe('the script import inside the shot list', () => {
|
||||
it.each(CLOSERS.slice(0, 4))('closing it with %s leaves the shot list and the job form open', async (_name, close) => {
|
||||
const user = userEvent.setup();
|
||||
await renderH3Modal(user);
|
||||
await user.click(await screen.findByRole('button', { name: 'Open shot list' }));
|
||||
const shotList = await screen.findByRole('dialog', { name: 'Shot list' });
|
||||
await user.click(within(shotList).getByRole('button', { name: 'Import script…' }));
|
||||
const importDialog = await screen.findByRole('dialog', { name: 'Import script' });
|
||||
|
||||
await close(user, importDialog);
|
||||
|
||||
await waitFor(() => expect(screen.queryByRole('dialog', { name: 'Import script' })).not.toBeInTheDocument());
|
||||
expect(screen.getByRole('dialog', { name: 'Shot list' })).toBeInTheDocument();
|
||||
expectParentStillOpen();
|
||||
});
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,253 @@
|
||||
import { render, screen, within } from '@testing-library/react';
|
||||
import userEvent from '@testing-library/user-event';
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { H3ShotListEditor } from './H3ShotListEditor';
|
||||
|
||||
const TWO_SHOTS = '[Shot 1] Wide shot. A dock.\n[Shot 2] At 00:02.000, the shot cuts to a close-up shot. A face.';
|
||||
const WITH_LINE = '[Shot 1] Wide shot. (S1) <d>[English] Hi.</d>';
|
||||
|
||||
type User = ReturnType<typeof userEvent.setup>;
|
||||
|
||||
function setup(detailed: string, extra: { open?: boolean } = {}) {
|
||||
const onApply = vi.fn();
|
||||
const ui = (open: boolean) => (
|
||||
<H3ShotListEditor
|
||||
open={open}
|
||||
subjectDefinitions="<Subject 1> is a man."
|
||||
initialDetailedDescription={detailed}
|
||||
initialRetentionAnalysis=""
|
||||
onApply={onApply}
|
||||
onClose={() => {}}
|
||||
/>
|
||||
);
|
||||
const view = render(ui(extra.open ?? true));
|
||||
return { onApply, view, ui };
|
||||
}
|
||||
|
||||
const undoButton = () => screen.getByRole('button', { name: /Undo/ });
|
||||
const redoButton = () => screen.getByRole('button', { name: /Redo/ });
|
||||
const applied = (onApply: ReturnType<typeof vi.fn>) => onApply.mock.calls.at(-1)?.[0] as string;
|
||||
const apply = (user: User) => user.click(screen.getByRole('button', { name: 'Apply' }));
|
||||
const lineBox = () => screen.getByPlaceholderText('What the speaker says') as HTMLTextAreaElement;
|
||||
const ctrl = (key: string) => `{Control>}${key}{/Control}`;
|
||||
|
||||
describe('shot list undo/redo', () => {
|
||||
it('starts with nothing to undo or redo', () => {
|
||||
setup(TWO_SHOTS);
|
||||
expect(undoButton()).toBeDisabled();
|
||||
expect(redoButton()).toBeDisabled();
|
||||
});
|
||||
|
||||
it('undoes and redoes moving a shot', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup(TWO_SHOTS);
|
||||
|
||||
await user.click(screen.getAllByRole('button', { name: '↑' })[1]);
|
||||
expect(undoButton()).toBeEnabled();
|
||||
|
||||
await user.click(undoButton());
|
||||
expect(redoButton()).toBeEnabled();
|
||||
await apply(user);
|
||||
expect(applied(onApply)).toBe(TWO_SHOTS);
|
||||
});
|
||||
|
||||
it('redo reapplies what was undone', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup(TWO_SHOTS);
|
||||
await user.click(screen.getAllByRole('button', { name: '↑' })[1]);
|
||||
await user.click(undoButton());
|
||||
await user.click(redoButton());
|
||||
await apply(user);
|
||||
expect(applied(onApply).startsWith('[Shot 1] Close-up shot. A face.')).toBe(true);
|
||||
});
|
||||
|
||||
it('undoes removing a shot, dialogue and all', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup(`${WITH_LINE}\n[Shot 2] At 00:02.500, the shot cuts to a wide shot. B.`);
|
||||
await user.click(screen.getAllByRole('button', { name: 'Remove' })[0]);
|
||||
expect(screen.queryByPlaceholderText('What the speaker says')).not.toBeInTheDocument();
|
||||
|
||||
await user.click(undoButton());
|
||||
expect(lineBox().value).toBe('Hi.');
|
||||
await apply(user);
|
||||
expect(applied(onApply)).toContain('(S1) <d>[English] Hi.</d>');
|
||||
});
|
||||
|
||||
it('undoes an add, a type change, a subject toggle, and a cut time in turn', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup(TWO_SHOTS);
|
||||
|
||||
await user.click(screen.getAllByRole('button', { name: /Add shot/ })[0]);
|
||||
await user.selectOptions(screen.getAllByLabelText('Shot type')[0], 'pov');
|
||||
await user.click(screen.getAllByRole('button', { name: '<Subject 1>' })[0]);
|
||||
const cut = screen.getAllByPlaceholderText(/\d\d:\d\d\.\d{3}/)[0] as HTMLInputElement;
|
||||
await user.type(cut, '5{Enter}');
|
||||
|
||||
// four discrete steps, undone one at a time
|
||||
for (let i = 0; i < 4; i += 1) await user.click(undoButton());
|
||||
expect(undoButton()).toBeDisabled();
|
||||
await apply(user);
|
||||
expect(applied(onApply)).toBe(TWO_SHOTS);
|
||||
});
|
||||
|
||||
it('committing a cut time that has not changed is not a step', async () => {
|
||||
const user = userEvent.setup();
|
||||
setup(TWO_SHOTS);
|
||||
const cut = screen.getAllByPlaceholderText(/\d\d:\d\d\.\d{3}/)[0] as HTMLInputElement;
|
||||
await user.type(cut, '5{Enter}');
|
||||
await user.click(undoButton()); // the click blurs the box, which re-commits 00:05.000
|
||||
expect(undoButton()).toBeDisabled();
|
||||
});
|
||||
|
||||
it('a burst of typing is one step', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup(WITH_LINE);
|
||||
await user.type(lineBox(), ' there my friend');
|
||||
expect(lineBox().value).toBe('Hi. there my friend');
|
||||
|
||||
await user.click(undoButton());
|
||||
expect(lineBox().value).toBe('Hi.');
|
||||
expect(undoButton()).toBeDisabled();
|
||||
await apply(user);
|
||||
expect(applied(onApply)).toContain('<d>[English] Hi.</d>');
|
||||
});
|
||||
|
||||
it('a tag inserted from the palette is its own step, separate from typing', async () => {
|
||||
const user = userEvent.setup();
|
||||
setup(WITH_LINE);
|
||||
await user.type(lineBox(), ' there');
|
||||
lineBox().setSelectionRange(lineBox().value.length, lineBox().value.length);
|
||||
await user.click(screen.getByRole('button', { name: /<pause>/ }));
|
||||
expect(lineBox().value).toBe('Hi. there <pause>');
|
||||
|
||||
await user.click(undoButton());
|
||||
expect(lineBox().value).toBe('Hi. there');
|
||||
await user.click(undoButton());
|
||||
expect(lineBox().value).toBe('Hi.');
|
||||
});
|
||||
|
||||
it('undoes edits to the description and the scene notes', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup('[Shot 1] Wide shot. A dock.');
|
||||
await user.type(screen.getByPlaceholderText(/Live-action, cinematic/), 'Noir.');
|
||||
await user.type(screen.getByPlaceholderText("What's happening in this shot"), ' At dawn.');
|
||||
|
||||
await user.click(undoButton());
|
||||
await user.click(undoButton());
|
||||
expect(undoButton()).toBeDisabled();
|
||||
await apply(user);
|
||||
expect(applied(onApply)).toBe('[Shot 1] Wide shot. A dock.');
|
||||
});
|
||||
|
||||
it('typing in different boxes makes separate steps', async () => {
|
||||
const user = userEvent.setup();
|
||||
setup('[Shot 1] Wide shot. A dock.');
|
||||
await user.type(screen.getByPlaceholderText(/Live-action, cinematic/), 'Noir.');
|
||||
await user.type(screen.getByPlaceholderText("What's happening in this shot"), ' At dawn.');
|
||||
|
||||
await user.click(undoButton());
|
||||
expect((screen.getByPlaceholderText("What's happening in this shot") as HTMLTextAreaElement).value).toBe('A dock.');
|
||||
expect((screen.getByPlaceholderText(/Live-action, cinematic/) as HTMLTextAreaElement).value).toBe('Noir.');
|
||||
});
|
||||
|
||||
it('a new edit clears what could be redone', async () => {
|
||||
const user = userEvent.setup();
|
||||
setup(TWO_SHOTS);
|
||||
await user.click(screen.getAllByRole('button', { name: '↑' })[1]);
|
||||
await user.click(undoButton());
|
||||
expect(redoButton()).toBeEnabled();
|
||||
await user.click(screen.getAllByRole('button', { name: /Add shot/ })[0]);
|
||||
expect(redoButton()).toBeDisabled();
|
||||
});
|
||||
});
|
||||
|
||||
describe('shot list undo/redo shortcuts', () => {
|
||||
it('Ctrl+Z from inside a textbox undoes the whole list, not just that box', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup(TWO_SHOTS);
|
||||
await user.click(screen.getAllByRole('button', { name: '↑' })[1]); // move a shot
|
||||
// focus a different textbox, then undo from there
|
||||
const box = screen.getAllByPlaceholderText("What's happening in this shot")[0] as HTMLTextAreaElement;
|
||||
await user.click(box);
|
||||
await user.keyboard(ctrl('z'));
|
||||
expect(undoButton()).toBeDisabled();
|
||||
await apply(user);
|
||||
expect(applied(onApply)).toBe(TWO_SHOTS);
|
||||
});
|
||||
|
||||
it('redoes with Ctrl+Shift+Z and with Ctrl+Y', async () => {
|
||||
const user = userEvent.setup();
|
||||
setup(TWO_SHOTS);
|
||||
await user.click(screen.getAllByRole('button', { name: '↑' })[1]);
|
||||
const box = screen.getAllByPlaceholderText("What's happening in this shot")[0];
|
||||
await user.click(box);
|
||||
|
||||
await user.keyboard(ctrl('z'));
|
||||
expect(redoButton()).toBeEnabled();
|
||||
await user.keyboard('{Control>}{Shift>}z{/Shift}{/Control}');
|
||||
expect(redoButton()).toBeDisabled();
|
||||
|
||||
await user.keyboard(ctrl('z'));
|
||||
await user.keyboard(ctrl('y'));
|
||||
expect(redoButton()).toBeDisabled();
|
||||
expect(undoButton()).toBeEnabled();
|
||||
});
|
||||
|
||||
it('Cmd+Z works too', async () => {
|
||||
const user = userEvent.setup();
|
||||
setup(TWO_SHOTS);
|
||||
await user.click(screen.getAllByRole('button', { name: '↑' })[1]);
|
||||
await user.click(screen.getAllByPlaceholderText("What's happening in this shot")[0]);
|
||||
await user.keyboard('{Meta>}z{/Meta}');
|
||||
expect(undoButton()).toBeDisabled();
|
||||
});
|
||||
|
||||
it('leaves the script import box to its own undo', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup(TWO_SHOTS);
|
||||
await user.click(screen.getAllByRole('button', { name: '↑' })[1]);
|
||||
expect(undoButton()).toBeEnabled();
|
||||
|
||||
await user.click(screen.getByRole('button', { name: 'Import script…' }));
|
||||
const dialog = await screen.findByRole('dialog', { name: 'Import script' });
|
||||
const box = within(dialog).getByLabelText('Script');
|
||||
await user.click(box);
|
||||
await user.keyboard(ctrl('z'));
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Cancel' }));
|
||||
|
||||
// the shot list's history was untouched: the move is still there to undo
|
||||
await user.click(await screen.findByRole('button', { name: /Undo/ }));
|
||||
await apply(user);
|
||||
expect(applied(onApply)).toBe(TWO_SHOTS);
|
||||
});
|
||||
});
|
||||
|
||||
describe('shot list history across imports and reopening', () => {
|
||||
it('loading a script clip is one undoable step', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { onApply } = setup(TWO_SHOTS);
|
||||
await user.click(screen.getByRole('button', { name: 'Import script…' }));
|
||||
const dialog = await screen.findByRole('dialog', { name: 'Import script' });
|
||||
await user.click(within(dialog).getByLabelText('Script'));
|
||||
await user.paste('CAPTAIN: Sit down and hear me out.\n\nCLERK: Yes, sir.');
|
||||
await user.click(within(dialog).getByRole('button', { name: 'Load clip' }));
|
||||
await screen.findAllByPlaceholderText('What the speaker says');
|
||||
|
||||
await user.click(undoButton());
|
||||
expect(screen.queryByPlaceholderText('What the speaker says')).not.toBeInTheDocument();
|
||||
await apply(user);
|
||||
expect(applied(onApply)).toBe(TWO_SHOTS);
|
||||
});
|
||||
|
||||
it('opening the editor again starts a fresh history', async () => {
|
||||
const user = userEvent.setup();
|
||||
const { view, ui } = setup(TWO_SHOTS);
|
||||
await user.click(screen.getAllByRole('button', { name: '↑' })[1]);
|
||||
expect(undoButton()).toBeEnabled();
|
||||
|
||||
view.rerender(ui(false));
|
||||
view.rerender(ui(true));
|
||||
expect(await screen.findByRole('button', { name: /Undo/ })).toBeDisabled();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,111 @@
|
||||
import { fireEvent, render, screen } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { BeforeAfterGrade } from './BeforeAfterGrade';
|
||||
|
||||
// jsdom has no real video decoder, canvas 2D context, or `ImageData` constructor,
|
||||
// so all three are stubbed: the video always "has" an 8x6 frame ready, the canvas
|
||||
// records what was drawn/read/written instead of actually rasterizing anything,
|
||||
// and ImageData is a plain data holder (its real job -- backing a canvas -- is
|
||||
// already covered by the getContext stub).
|
||||
if (typeof globalThis.ImageData === 'undefined') {
|
||||
class FakeImageData {
|
||||
data: Uint8ClampedArray;
|
||||
width: number;
|
||||
height: number;
|
||||
constructor(data: Uint8ClampedArray, width: number, height: number) {
|
||||
this.data = data;
|
||||
this.width = width;
|
||||
this.height = height;
|
||||
}
|
||||
}
|
||||
// @ts-expect-error -- test-only polyfill for a browser global jsdom doesn't provide
|
||||
globalThis.ImageData = FakeImageData;
|
||||
}
|
||||
|
||||
function stubVideoFrame() {
|
||||
Object.defineProperty(HTMLMediaElement.prototype, 'readyState', { configurable: true, get: () => 4 });
|
||||
Object.defineProperty(HTMLVideoElement.prototype, 'videoWidth', { configurable: true, get: () => 8 });
|
||||
Object.defineProperty(HTMLVideoElement.prototype, 'videoHeight', { configurable: true, get: () => 6 });
|
||||
}
|
||||
|
||||
function stubCanvasContext() {
|
||||
const data = new Uint8ClampedArray(8 * 6 * 4).fill(100);
|
||||
const putImageData = vi.fn();
|
||||
const drawImage = vi.fn();
|
||||
const getImageData = vi.fn(() => ({ width: 8, height: 6, data }));
|
||||
vi.spyOn(HTMLCanvasElement.prototype, 'getContext').mockReturnValue({
|
||||
drawImage,
|
||||
getImageData,
|
||||
putImageData,
|
||||
} as unknown as CanvasRenderingContext2D);
|
||||
return { drawImage, getImageData, putImageData, data };
|
||||
}
|
||||
|
||||
describe('BeforeAfterGrade', () => {
|
||||
beforeEach(() => {
|
||||
stubVideoFrame();
|
||||
});
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
it('shows the "load a frame" hint until a video frame is captured', () => {
|
||||
const ref = { current: null };
|
||||
render(<BeforeAfterGrade videoRef={ref} brightness={0} contrast={1} saturation={1} />);
|
||||
expect(screen.getByText(/Play or seek the preview above/)).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('captures a frame once the video has data, and draws it into both canvases', () => {
|
||||
const { getImageData, putImageData } = stubCanvasContext();
|
||||
const video = document.createElement('video');
|
||||
document.body.appendChild(video);
|
||||
const ref = { current: video };
|
||||
|
||||
render(<BeforeAfterGrade videoRef={ref} brightness={0} contrast={1} saturation={1} />);
|
||||
fireEvent.loadedData(video);
|
||||
|
||||
expect(getImageData).toHaveBeenCalled();
|
||||
expect(putImageData).toHaveBeenCalled();
|
||||
expect(screen.queryByText(/Play or seek the preview above/)).not.toBeInTheDocument();
|
||||
document.body.removeChild(video);
|
||||
});
|
||||
|
||||
it('re-grades the "after" canvas when the color values change, without touching the video again', () => {
|
||||
const { putImageData } = stubCanvasContext();
|
||||
const video = document.createElement('video');
|
||||
document.body.appendChild(video);
|
||||
const ref = { current: video };
|
||||
|
||||
const { rerender } = render(<BeforeAfterGrade videoRef={ref} brightness={0} contrast={1} saturation={1} />);
|
||||
fireEvent.loadedData(video);
|
||||
const callsAfterCapture = putImageData.mock.calls.length;
|
||||
|
||||
rerender(<BeforeAfterGrade videoRef={ref} brightness={50} contrast={1} saturation={1} />);
|
||||
expect(putImageData.mock.calls.length).toBeGreaterThan(callsAfterCapture);
|
||||
|
||||
// Neutral values were drawn as-is; the brightened redraw's pixel buffer must differ.
|
||||
const neutralData = putImageData.mock.calls[0][0].data as Uint8ClampedArray;
|
||||
const brightenedData = putImageData.mock.calls[putImageData.mock.calls.length - 1][0].data as Uint8ClampedArray;
|
||||
expect(brightenedData[0]).toBeGreaterThan(neutralData[0]);
|
||||
|
||||
document.body.removeChild(video);
|
||||
});
|
||||
|
||||
it('"Use current frame" re-captures the video at its current position', () => {
|
||||
const { drawImage } = stubCanvasContext();
|
||||
const video = document.createElement('video');
|
||||
document.body.appendChild(video);
|
||||
const ref = { current: video };
|
||||
|
||||
render(<BeforeAfterGrade videoRef={ref} brightness={0} contrast={1} saturation={1} />);
|
||||
fireEvent.loadedData(video);
|
||||
const callsAfterAutoCapture = drawImage.mock.calls.length;
|
||||
|
||||
fireEvent.click(screen.getByRole('button', { name: 'Use current frame' }));
|
||||
expect(drawImage.mock.calls.length).toBeGreaterThan(callsAfterAutoCapture);
|
||||
|
||||
document.body.removeChild(video);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,113 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { applyGradeInPlace, isNeutralGrade } from '@/lib/grading';
|
||||
|
||||
export interface BeforeAfterGradeProps {
|
||||
/** the video the snapshot is captured from -- must be `crossOrigin="anonymous"` for pixel access to work */
|
||||
videoRef: React.RefObject<HTMLVideoElement | null>;
|
||||
brightness: number;
|
||||
contrast: number;
|
||||
saturation: number;
|
||||
/** called with the video's currentTime whenever a frame is (re)captured, so a caller with multiple
|
||||
* sections can tell which one the previewed frame falls into and keep its own selection in sync */
|
||||
onFrameCaptured?: (timeSeconds: number) => void;
|
||||
}
|
||||
|
||||
// Downscaled so the per-pixel pass stays instant on every slider tick.
|
||||
const PREVIEW_WIDTH = 320;
|
||||
|
||||
/**
|
||||
* A still frame from `videoRef`, shown twice: untouched, and with the current
|
||||
* color values applied via `applyGradeInPlace` -- entirely in the browser, so
|
||||
* dragging a slider never waits on the server. Captures a frame automatically
|
||||
* once the video has one to show; "Use current frame" re-captures whatever the
|
||||
* video is currently paused/seeked to, for comparing a specific moment.
|
||||
*/
|
||||
export function BeforeAfterGrade({ videoRef, brightness, contrast, saturation, onFrameCaptured }: BeforeAfterGradeProps) {
|
||||
const beforeRef = React.useRef<HTMLCanvasElement>(null);
|
||||
const afterRef = React.useRef<HTMLCanvasElement>(null);
|
||||
const sourceRef = React.useRef<ImageData | null>(null);
|
||||
const [ready, setReady] = React.useState(false);
|
||||
// Kept in a ref, not a `capture` dependency, so an inline arrow function prop
|
||||
// doesn't force the video-listener effect below to tear down and re-add itself every render.
|
||||
const onFrameCapturedRef = React.useRef(onFrameCaptured);
|
||||
onFrameCapturedRef.current = onFrameCaptured;
|
||||
|
||||
const capture = React.useCallback(() => {
|
||||
const video = videoRef.current;
|
||||
const before = beforeRef.current;
|
||||
if (!video || !before || video.readyState < 2 || !video.videoWidth) return;
|
||||
|
||||
const scale = PREVIEW_WIDTH / video.videoWidth;
|
||||
const width = PREVIEW_WIDTH;
|
||||
const height = Math.round(video.videoHeight * scale);
|
||||
before.width = width;
|
||||
before.height = height;
|
||||
const ctx = before.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.drawImage(video, 0, 0, width, height);
|
||||
try {
|
||||
sourceRef.current = ctx.getImageData(0, 0, width, height);
|
||||
setReady(true);
|
||||
onFrameCapturedRef.current?.(video.currentTime);
|
||||
} catch {
|
||||
// A cross-origin frame the browser wouldn't expose pixels for -- comparison stays unavailable.
|
||||
sourceRef.current = null;
|
||||
setReady(false);
|
||||
}
|
||||
}, [videoRef]);
|
||||
|
||||
React.useEffect(() => {
|
||||
const video = videoRef.current;
|
||||
if (!video) return;
|
||||
if (video.readyState >= 2) capture();
|
||||
video.addEventListener('loadeddata', capture);
|
||||
video.addEventListener('seeked', capture);
|
||||
return () => {
|
||||
video.removeEventListener('loadeddata', capture);
|
||||
video.removeEventListener('seeked', capture);
|
||||
};
|
||||
}, [videoRef, capture]);
|
||||
|
||||
React.useEffect(() => {
|
||||
const source = sourceRef.current;
|
||||
const after = afterRef.current;
|
||||
if (!source || !after) return;
|
||||
after.width = source.width;
|
||||
after.height = source.height;
|
||||
const ctx = after.getContext('2d');
|
||||
if (!ctx) return;
|
||||
const copy = new ImageData(new Uint8ClampedArray(source.data), source.width, source.height);
|
||||
if (!isNeutralGrade(brightness, contrast, saturation)) {
|
||||
applyGradeInPlace(copy.data, brightness, contrast, saturation);
|
||||
}
|
||||
ctx.putImageData(copy, 0, 0);
|
||||
// eslint-disable-next-line react-hooks/exhaustive-deps -- redraws whenever a new frame is captured (`ready`) too
|
||||
}, [brightness, contrast, saturation, ready]);
|
||||
|
||||
return (
|
||||
<div className="flex flex-col gap-2">
|
||||
<div className="flex items-center justify-between">
|
||||
<p className="text-xs font-medium uppercase tracking-wide text-muted-foreground">Before / after</p>
|
||||
<button type="button" onClick={capture} className="text-xs text-accent-blue hover:underline">
|
||||
Use current frame
|
||||
</button>
|
||||
</div>
|
||||
<div className="grid grid-cols-2 gap-2">
|
||||
<div className="flex flex-col gap-1">
|
||||
<canvas ref={beforeRef} role="img" aria-label="Before color grading" className="w-full rounded-md border border-border bg-black" />
|
||||
<span className="text-center text-[11px] text-muted-foreground">Before</span>
|
||||
</div>
|
||||
<div className="flex flex-col gap-1">
|
||||
<canvas ref={afterRef} role="img" aria-label="After color grading" className="w-full rounded-md border border-border bg-black" />
|
||||
<span className="text-center text-[11px] text-muted-foreground">After</span>
|
||||
</div>
|
||||
</div>
|
||||
{!ready && (
|
||||
<p className="text-xs text-muted-foreground">Play or seek the preview above to load a frame to compare.</p>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { toast } from 'sonner';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { downloadMergedVideo, getMergedVideoUrl, type MergedScene } from '@/lib/api';
|
||||
import { formatClock } from '@/lib/scenes';
|
||||
import { downloadBlob } from '@/lib/utils';
|
||||
|
||||
export interface MergedSceneDialogProps {
|
||||
/** the merged video to show, or null when closed */
|
||||
merged: (MergedScene & { title: string }) | null;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
/** The finished, merged scene: play it, and save it. */
|
||||
export function MergedSceneDialog({ merged, onClose }: MergedSceneDialogProps) {
|
||||
const [downloading, setDownloading] = React.useState(false);
|
||||
|
||||
async function download() {
|
||||
if (!merged) return;
|
||||
setDownloading(true);
|
||||
try {
|
||||
downloadBlob(await downloadMergedVideo(merged.filename), merged.filename);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't download the video.");
|
||||
} finally {
|
||||
setDownloading(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={!!merged} onOpenChange={(open) => !open && onClose()}>
|
||||
<DialogContent className="max-w-3xl">
|
||||
<DialogHeader>
|
||||
<DialogTitle>{merged?.title} merged</DialogTitle>
|
||||
<DialogDescription>
|
||||
{merged &&
|
||||
`${merged.clips} clip${merged.clips === 1 ? '' : 's'} joined in order · ${formatClock(merged.seconds)}`}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
{merged && (
|
||||
<video
|
||||
src={getMergedVideoUrl(merged.filename)}
|
||||
aria-label={`Merged video of ${merged.title}`}
|
||||
className="max-h-[60vh] w-full rounded-md border border-border bg-black"
|
||||
controls
|
||||
playsInline
|
||||
preload="metadata"
|
||||
/>
|
||||
)}
|
||||
<DialogFooter>
|
||||
<Button type="button" variant="outline" onClick={onClose}>
|
||||
Close
|
||||
</Button>
|
||||
<Button type="button" onClick={download} disabled={downloading}>
|
||||
{downloading ? 'Downloading…' : 'Download'}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,230 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { Loader2 } from 'lucide-react';
|
||||
import { toast } from 'sonner';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import {
|
||||
duplicateJob,
|
||||
getClipRevision,
|
||||
startClipRevision,
|
||||
updateJob,
|
||||
type ClipRevision,
|
||||
type DraftedJob,
|
||||
} from '@/lib/api';
|
||||
import { formatDuration } from '@/lib/jobTiming';
|
||||
import { diffLines } from '@/lib/lineDiff';
|
||||
import type { SceneClip } from '@/lib/scenes';
|
||||
import type { Job } from '@/lib/types';
|
||||
|
||||
const POLL_MS = 3000;
|
||||
|
||||
function seconds(job: Pick<DraftedJob, 'num_frames' | 'fps'>): string {
|
||||
return `${((job.num_frames ?? 0) / (job.fps || 24)).toFixed(1)} s`;
|
||||
}
|
||||
|
||||
/** Changes outside the prompt text, as short sentences. */
|
||||
function settingChanges(before: DraftedJob, after: DraftedJob): string[] {
|
||||
const changes: string[] = [];
|
||||
if (before.num_frames !== after.num_frames) {
|
||||
changes.push(`Length ${seconds(before)} → ${seconds(after)} (${before.num_frames} → ${after.num_frames} frames)`);
|
||||
}
|
||||
const sources = (j: DraftedJob) => (j.references ?? []).map((r) => r.source);
|
||||
const was = sources(before);
|
||||
const now = sources(after);
|
||||
for (const s of now.filter((s) => !was.includes(s))) changes.push(`Added reference ${s.split('/').pop()}`);
|
||||
for (const s of was.filter((s) => !now.includes(s))) changes.push(`Removed reference ${s.split('/').pop()}`);
|
||||
if (was.length === now.length && was.some((s, i) => s !== now[i])) changes.push('Reordered the references');
|
||||
if (before.seed !== after.seed) changes.push(`Seed ${before.seed} → ${after.seed}`);
|
||||
return changes;
|
||||
}
|
||||
|
||||
export interface ReviseClipDialogProps {
|
||||
/** the clip to revise; its shown take is the one revised */
|
||||
clip: SceneClip | null;
|
||||
/** the shown takes of the clips before and after it, for continuity */
|
||||
neighbors: Job[];
|
||||
onClose: () => void;
|
||||
/** called after the revision is created as a new take */
|
||||
onCreated: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Describe what went wrong in a clip's take; the scene writer revises the
|
||||
* clip's job, and the revision becomes a new pending take of the same clip
|
||||
* (same name), leaving the old take as it is.
|
||||
*/
|
||||
export function ReviseClipDialog({ clip, neighbors, onClose, onCreated }: ReviseClipDialogProps) {
|
||||
const [feedback, setFeedback] = React.useState('');
|
||||
const [revision, setRevision] = React.useState<ClipRevision | null>(null);
|
||||
const [starting, setStarting] = React.useState(false);
|
||||
const [creating, setCreating] = React.useState(false);
|
||||
const [now, setNow] = React.useState(() => Date.now());
|
||||
|
||||
// Hiding the dialog keeps a running revision; opening it for a different take starts fresh.
|
||||
const jobId = clip?.job.id;
|
||||
const lastJobId = React.useRef<string | undefined>(undefined);
|
||||
React.useEffect(() => {
|
||||
if (jobId === undefined || jobId === lastJobId.current) return;
|
||||
lastJobId.current = jobId;
|
||||
setFeedback('');
|
||||
setRevision(null);
|
||||
}, [jobId]);
|
||||
|
||||
const running = revision?.status === 'running';
|
||||
React.useEffect(() => {
|
||||
if (!running || !revision) return;
|
||||
const runId = revision.run_id;
|
||||
const poll = setInterval(async () => {
|
||||
try {
|
||||
setRevision(await getClipRevision(runId));
|
||||
} catch {
|
||||
// Keep the last state; the next poll retries.
|
||||
}
|
||||
}, POLL_MS);
|
||||
const tick = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => {
|
||||
clearInterval(poll);
|
||||
clearInterval(tick);
|
||||
};
|
||||
}, [running, revision?.run_id]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
if (!clip) return null;
|
||||
const { job } = clip;
|
||||
|
||||
async function start() {
|
||||
setStarting(true);
|
||||
try {
|
||||
setRevision(await startClipRevision({ job, feedback, neighbors }));
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't start the revision.");
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function createTake() {
|
||||
if (!revision?.job) return;
|
||||
setCreating(true);
|
||||
try {
|
||||
const { id } = await duplicateJob(job.id);
|
||||
await updateJob(id, {
|
||||
prompt: revision.job.prompt.join('\n'),
|
||||
num_frames: revision.job.num_frames,
|
||||
references: revision.job.references ?? [],
|
||||
seed: revision.job.seed,
|
||||
});
|
||||
toast.success(`Created a new take of ${job.name}. It is pending; queue it when ready.`);
|
||||
onCreated();
|
||||
onClose();
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't create the take.");
|
||||
} finally {
|
||||
setCreating(false);
|
||||
}
|
||||
}
|
||||
|
||||
const done = revision?.status === 'completed' && revision.job !== null;
|
||||
const elapsed = revision ? (revision.finished_at ?? now / 1000) - revision.started_at : 0;
|
||||
const diff = done ? diffLines(revision.original.prompt.join('\n'), revision.job!.prompt.join('\n')) : [];
|
||||
const changes = done ? settingChanges(revision.original, revision.job!) : [];
|
||||
|
||||
return (
|
||||
<Dialog open onOpenChange={(o) => !o && !creating && onClose()}>
|
||||
<DialogContent className="max-h-[90vh] max-w-3xl overflow-y-auto">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Revise {job.name}</DialogTitle>
|
||||
<DialogDescription>
|
||||
Describe what went wrong in this take. The scene writer finds the cause in the prompt and rewrites it; you
|
||||
review the changes, then create them as a new take of the same clip. The current take is kept.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="flex flex-col gap-4 text-sm">
|
||||
<label className="flex flex-col gap-1">
|
||||
What went wrong
|
||||
<Textarea
|
||||
value={feedback}
|
||||
onChange={(e) => setFeedback(e.target.value)}
|
||||
rows={4}
|
||||
placeholder="e.g. There are 4 slaps instead of 2. / Andrew appears twice. / The band disappears after the chair throw."
|
||||
disabled={running}
|
||||
/>
|
||||
</label>
|
||||
|
||||
{revision && (
|
||||
<section aria-live="polite" className="flex flex-col gap-3 rounded-lg border border-border p-3">
|
||||
<div className="flex items-center gap-2">
|
||||
{running && <Loader2 className="size-4 animate-spin text-primary" aria-hidden />}
|
||||
<span className="font-medium">
|
||||
{running ? 'Revising…' : done ? 'Revision ready' : 'Revision failed'}
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">{formatDuration(elapsed)}</span>
|
||||
</div>
|
||||
{revision.error && <p className="text-xs text-destructive">{revision.error}</p>}
|
||||
{revision.final_response && (
|
||||
<p className="whitespace-pre-wrap text-xs">{revision.final_response}</p>
|
||||
)}
|
||||
{changes.length > 0 && (
|
||||
<ul className="list-disc pl-5 text-xs">
|
||||
{changes.map((c) => (
|
||||
<li key={c}>{c}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
{done && (
|
||||
<pre className="max-h-96 overflow-auto whitespace-pre-wrap rounded bg-background p-2 text-[11px]">
|
||||
{diff.map((line, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className={
|
||||
line.kind === 'added'
|
||||
? 'bg-green-500/15 text-green-700 dark:text-green-400'
|
||||
: line.kind === 'removed'
|
||||
? 'bg-red-500/15 text-red-700 line-through dark:text-red-400'
|
||||
: 'text-muted-foreground'
|
||||
}
|
||||
>
|
||||
{line.kind === 'added' ? '+ ' : line.kind === 'removed' ? '- ' : ' '}
|
||||
{line.text}
|
||||
</div>
|
||||
))}
|
||||
</pre>
|
||||
)}
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<DialogFooter>
|
||||
<Button type="button" variant="outline" onClick={onClose} disabled={creating}>
|
||||
{running ? 'Hide' : 'Cancel'}
|
||||
</Button>
|
||||
{done ? (
|
||||
<>
|
||||
<Button type="button" variant="outline" onClick={start} disabled={!feedback.trim() || creating}>
|
||||
Revise again
|
||||
</Button>
|
||||
<Button type="button" onClick={createTake} disabled={creating}>
|
||||
{creating ? 'Creating…' : 'Create new take'}
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<Button type="button" onClick={start} disabled={!feedback.trim() || running || starting}>
|
||||
{starting ? 'Starting…' : running ? 'Revising…' : 'Revise'}
|
||||
</Button>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,484 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { Timer } from 'lucide-react';
|
||||
|
||||
import { H3ShotPanel } from '@/components/jobs/H3ShotPanel';
|
||||
import { Badge, type BadgeProps } from '@/components/ui/badge';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { useJobElapsed } from '@/hooks/useJobElapsed';
|
||||
import { NativeSelect } from '@/components/ui/native-select';
|
||||
import { getJobVideoUrl } from '@/lib/api';
|
||||
import { attachedLastFrames, whyCannotAttach } from '@/lib/continuation';
|
||||
import { formatCutTime } from '@/lib/h3Shots';
|
||||
import { computeElapsed } from '@/lib/jobTiming';
|
||||
import { QUEUEABLE_STATUSES } from '@/lib/sceneQueue';
|
||||
import { formatClock, type Scene, type SceneClip } from '@/lib/scenes';
|
||||
import type { Job } from '@/lib/types';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
const BADGE_VARIANTS: Record<string, BadgeProps['variant']> = {
|
||||
pending: 'secondary',
|
||||
queued: 'default',
|
||||
running: 'warning',
|
||||
completed: 'success',
|
||||
failed: 'destructive',
|
||||
stopped: 'secondary',
|
||||
};
|
||||
|
||||
const SEGMENT_TONE: Record<string, string> = {
|
||||
completed: 'bg-emerald-500/70',
|
||||
running: 'bg-amber-500/70',
|
||||
queued: 'bg-sky-500/60',
|
||||
failed: 'bg-rose-500/70',
|
||||
};
|
||||
|
||||
/** Statuses a job can still be edited in (mirrors the Jobs page). */
|
||||
export const EDITABLE_STATUSES = ['pending', 'failed', 'stopped'];
|
||||
|
||||
function clipLabel(clip: SceneClip): string {
|
||||
return clip.job.name?.trim() || `Job ${clip.job.id.slice(0, 8)}`;
|
||||
}
|
||||
|
||||
function takeLabel(job: Job, index: number, total: number): string {
|
||||
const ms = job.created_at < 1e12 ? job.created_at * 1000 : job.created_at;
|
||||
const when = new Date(ms).toLocaleString(undefined, {
|
||||
month: 'short',
|
||||
day: 'numeric',
|
||||
hour: '2-digit',
|
||||
minute: '2-digit',
|
||||
});
|
||||
const ran = job.status === 'running' ? null : computeElapsed(job, 0);
|
||||
return `Take ${index + 1} of ${total} · ${job.status}${ran ? ` in ${ran}` : ''} · ${when}`;
|
||||
}
|
||||
|
||||
function scrollToClip(id: string) {
|
||||
document.getElementById(`clip-${id}`)?.scrollIntoView({ behavior: 'smooth', block: 'start' });
|
||||
}
|
||||
|
||||
function Timeline({ scene }: { scene: Scene }) {
|
||||
return (
|
||||
<div className="flex h-4 w-full gap-px overflow-hidden rounded-md" role="group" aria-label="Scene timeline">
|
||||
{scene.clips.map((clip) => (
|
||||
<button
|
||||
key={clip.job.id}
|
||||
type="button"
|
||||
onClick={() => scrollToClip(clip.job.id)}
|
||||
aria-label={`Go to clip ${clip.position}: ${clipLabel(clip)}`}
|
||||
title={`Clip ${clip.position} · ${clipLabel(clip)} · ${clip.job.status} · ${formatClock(clip.startSeconds)}–${formatClock(clip.startSeconds + clip.durationSeconds)}`}
|
||||
style={{ flexGrow: Math.max(clip.durationSeconds, 0.1), flexBasis: 0 }}
|
||||
className={cn(
|
||||
'min-w-[3px] transition-opacity hover:opacity-70',
|
||||
SEGMENT_TONE[clip.job.status] ?? 'bg-muted-foreground/40',
|
||||
)}
|
||||
/>
|
||||
))}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function ClipPreview({ job, videoVersion }: { job: Job; videoVersion: number }) {
|
||||
const isImage = job.output_path?.toLowerCase().endsWith('.png') ?? false;
|
||||
const src = getJobVideoUrl(job.id, videoVersion);
|
||||
// Sets the *initial* muted state only, imperatively, so it doesn't fight the
|
||||
// user's own unmute via the native controls: `muted` as a plain JSX prop is
|
||||
// controlled, and React re-applies it on every re-render of this component
|
||||
// (e.g. the scene board's poll while a job is queued/running), silently
|
||||
// re-muting a video the user just unmuted.
|
||||
const videoRef = React.useRef<HTMLVideoElement | null>(null);
|
||||
React.useEffect(() => {
|
||||
if (videoRef.current) videoRef.current.muted = true;
|
||||
}, []);
|
||||
return isImage ? (
|
||||
// eslint-disable-next-line @next/next/no-img-element
|
||||
<img src={src} alt={job.name ?? 'Generated output'} className="max-h-56 rounded-md border border-border" />
|
||||
) : (
|
||||
<video
|
||||
ref={videoRef}
|
||||
src={src}
|
||||
aria-label={`Generated video for ${job.name ?? job.id}`}
|
||||
className="max-h-56 rounded-md border border-border bg-black"
|
||||
controls
|
||||
loop
|
||||
playsInline
|
||||
preload="metadata"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
function ClipRow({
|
||||
clip,
|
||||
previous,
|
||||
firstShotNumber,
|
||||
onOpenJob,
|
||||
onChooseTake,
|
||||
onUseLastFrame,
|
||||
onDetachLastFrame,
|
||||
onQueueClip,
|
||||
onDequeueClip,
|
||||
onTrimClip,
|
||||
onReviseClip,
|
||||
videoVersionFor,
|
||||
}: {
|
||||
clip: SceneClip;
|
||||
/** the clip before this one in the scene, if any */
|
||||
previous: SceneClip | undefined;
|
||||
firstShotNumber: number;
|
||||
onOpenJob: (job: Job) => void;
|
||||
onChooseTake: (takeKey: string, jobId: string) => void;
|
||||
onUseLastFrame: (clip: SceneClip, previous: SceneClip) => Promise<void>;
|
||||
onDetachLastFrame: (clip: SceneClip) => Promise<void>;
|
||||
onQueueClip: (clip: SceneClip) => Promise<void>;
|
||||
onDequeueClip: (clip: SceneClip) => Promise<void>;
|
||||
onTrimClip: (clip: SceneClip) => void;
|
||||
onReviseClip: (clip: SceneClip) => void;
|
||||
videoVersionFor: (jobId: string) => number;
|
||||
}) {
|
||||
const [previewing, setPreviewing] = React.useState(false);
|
||||
const [attaching, setAttaching] = React.useState(false);
|
||||
const [queueing, setQueueing] = React.useState(false);
|
||||
const { job } = clip;
|
||||
const elapsed = useJobElapsed(job);
|
||||
const canPreview = job.status === 'completed' && !!job.output_path;
|
||||
const editable = EDITABLE_STATUSES.includes(job.status);
|
||||
|
||||
const queued = job.status === 'queued';
|
||||
const canQueue = QUEUEABLE_STATUSES.includes(job.status);
|
||||
|
||||
// Starting from the previous clip's last frame: offered on clips that can still be
|
||||
// edited. The frame is taken when this clip runs, so the previous clip needn't be done yet.
|
||||
const frames = attachedLastFrames(job.references);
|
||||
const fromPrevious = previous && frames.some((f) => f.jobId === previous.job.id);
|
||||
const fromAnotherTake = frames.some((f) => f.jobId !== previous?.job.id);
|
||||
const previousDone = !!previous && previous.job.status === 'completed';
|
||||
const cannotUseFrame = previous ? whyCannotAttach(job.prompt, job.references) : null;
|
||||
|
||||
const usingLastFrame = frames.length > 0;
|
||||
const lastFrameBlocked = attaching || (!usingLastFrame && !!cannotUseFrame);
|
||||
|
||||
async function toggleLastFrame() {
|
||||
if (!previous) return;
|
||||
setAttaching(true);
|
||||
try {
|
||||
await (usingLastFrame ? onDetachLastFrame(clip) : onUseLastFrame(clip, previous));
|
||||
} finally {
|
||||
setAttaching(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function queueOrDequeue(action: (clip: SceneClip) => Promise<void>) {
|
||||
setQueueing(true);
|
||||
try {
|
||||
await action(clip);
|
||||
} finally {
|
||||
setQueueing(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<li id={`clip-${job.id}`} className="scroll-mt-32 rounded-xl border border-border bg-card p-3">
|
||||
<div className="mb-2 flex flex-wrap items-center gap-2">
|
||||
<span className="flex size-6 shrink-0 items-center justify-center rounded-md bg-secondary text-xs font-semibold text-foreground">
|
||||
{clip.position}
|
||||
</span>
|
||||
<h3 className="min-w-0 truncate text-sm font-semibold text-foreground">{clipLabel(clip)}</h3>
|
||||
<Badge variant={BADGE_VARIANTS[job.status] ?? 'secondary'}>{job.status}</Badge>
|
||||
{elapsed && (
|
||||
<span
|
||||
className="inline-flex items-center gap-1 text-xs text-muted-foreground"
|
||||
title={job.status === 'running' ? 'Running for' : 'Took'}
|
||||
>
|
||||
<Timer className="size-3.5" aria-hidden />
|
||||
{elapsed}
|
||||
</span>
|
||||
)}
|
||||
<span className="font-mono text-xs text-muted-foreground">
|
||||
{formatClock(clip.startSeconds)}–{formatClock(clip.startSeconds + clip.durationSeconds)}
|
||||
</span>
|
||||
{clip.continuation && (
|
||||
<Badge variant="outline" className="normal-case tracking-normal">
|
||||
continues previous shot
|
||||
</Badge>
|
||||
)}
|
||||
<span className="ml-auto flex flex-wrap items-center justify-end gap-2">
|
||||
{previous && queued && fromPrevious && (
|
||||
<Badge variant="outline" className="normal-case tracking-normal">
|
||||
starts from clip {previous.position}'s last frame
|
||||
</Badge>
|
||||
)}
|
||||
{previous && queued && fromPrevious && !previousDone && (
|
||||
<Badge variant="outline" className="normal-case tracking-normal">
|
||||
waiting for clip {previous.position}
|
||||
</Badge>
|
||||
)}
|
||||
{previous && editable && (
|
||||
<label
|
||||
className={cn(
|
||||
'flex items-center gap-1.5 text-xs text-foreground',
|
||||
lastFrameBlocked && 'text-muted-foreground',
|
||||
)}
|
||||
title={
|
||||
usingLastFrame
|
||||
? "Stop starting this clip from the previous clip's last frame"
|
||||
: (cannotUseFrame ??
|
||||
`Attach the last frame of clip ${previous.position} as this clip's opening frame${
|
||||
previousDone ? '' : ` (taken when this clip runs, after clip ${previous.position} has finished)`
|
||||
}`)
|
||||
}
|
||||
>
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={usingLastFrame}
|
||||
onChange={toggleLastFrame}
|
||||
disabled={lastFrameBlocked}
|
||||
className="size-4"
|
||||
/>
|
||||
{attaching
|
||||
? usingLastFrame
|
||||
? 'Removing…'
|
||||
: 'Attaching…'
|
||||
: fromPrevious || !usingLastFrame
|
||||
? `Use last frame of clip ${previous.position}`
|
||||
: "Starts from another take's last frame"}
|
||||
</label>
|
||||
)}
|
||||
{clip.takeKey && clip.takes.length > 1 && (
|
||||
<NativeSelect
|
||||
aria-label={`Take for clip ${clip.position}`}
|
||||
value={job.id}
|
||||
onChange={(e) => onChooseTake(clip.takeKey as string, e.target.value)}
|
||||
className="h-8 w-auto py-0 text-xs"
|
||||
>
|
||||
{clip.takes.map((take, i) => (
|
||||
<option key={take.id} value={take.id}>
|
||||
{takeLabel(take, i, clip.takes.length)}
|
||||
</option>
|
||||
))}
|
||||
</NativeSelect>
|
||||
)}
|
||||
{canPreview && (
|
||||
<Button type="button" size="sm" variant="outline" onClick={() => setPreviewing((p) => !p)}>
|
||||
{previewing ? 'Hide video' : 'Preview'}
|
||||
</Button>
|
||||
)}
|
||||
{canPreview && (
|
||||
<Button type="button" size="sm" variant="outline" onClick={() => onTrimClip(clip)}>
|
||||
Edit video
|
||||
</Button>
|
||||
)}
|
||||
{canPreview && (
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => onReviseClip(clip)}
|
||||
title="Describe what went wrong; the scene writer rewrites the prompt as a new take"
|
||||
>
|
||||
Revise
|
||||
</Button>
|
||||
)}
|
||||
{canQueue && (
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => queueOrDequeue(onQueueClip)}
|
||||
disabled={queueing}
|
||||
title="Run this clip when the ones queued before it have finished"
|
||||
>
|
||||
Queue
|
||||
</Button>
|
||||
)}
|
||||
{queued && (
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => queueOrDequeue(onDequeueClip)}
|
||||
disabled={queueing}
|
||||
>
|
||||
Remove from queue
|
||||
</Button>
|
||||
)}
|
||||
<Button type="button" size="sm" variant="outline" onClick={() => onOpenJob(job)}>
|
||||
{editable ? 'Edit' : 'View'}
|
||||
</Button>
|
||||
</span>
|
||||
</div>
|
||||
|
||||
{previewing && canPreview && (
|
||||
<div className="mb-2">
|
||||
<ClipPreview job={job} videoVersion={videoVersionFor(job.id)} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
{clip.shots.length === 0 ? (
|
||||
<p className="rounded-lg border border-dashed border-border p-3 text-sm text-muted-foreground">
|
||||
This job's prompt has no shots.
|
||||
</p>
|
||||
) : (
|
||||
<div className="flex items-stretch gap-2 overflow-x-auto pb-1" aria-label={`Shots in ${clipLabel(clip)}`}>
|
||||
{clip.shots.map((shot, i) => (
|
||||
<div
|
||||
key={shot.id}
|
||||
className="flex w-52 shrink-0 flex-col gap-1.5 rounded-xl border border-border bg-background p-2"
|
||||
>
|
||||
<H3ShotPanel
|
||||
shot={shot}
|
||||
number={firstShotNumber + i}
|
||||
timeLabel={formatCutTime(clip.shotStarts[i])}
|
||||
/>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</li>
|
||||
);
|
||||
}
|
||||
|
||||
export interface SceneBoardProps {
|
||||
scene: Scene;
|
||||
onOpenJob: (job: Job) => void;
|
||||
/** start a clip from the last frame of the one before it; resolves when done (errors are reported by the caller) */
|
||||
onUseLastFrame: (clip: SceneClip, previous: SceneClip) => Promise<void>;
|
||||
/** pick which re-run of a clip number to show */
|
||||
onChooseTake: (takeKey: string, jobId: string) => void;
|
||||
/** open the confirmation for queueing every runnable clip of the scene */
|
||||
onQueueScene: (scene: Scene) => void;
|
||||
/** join the scene's finished clips into one video; resolves when done (errors are reported by the caller) */
|
||||
onMergeScene: (scene: Scene) => Promise<void>;
|
||||
/** stop starting a clip from a previous clip's last frame; resolves when done */
|
||||
onDetachLastFrame: (clip: SceneClip) => Promise<void>;
|
||||
/** queue / unqueue one clip; resolve when done (errors are reported by the caller) */
|
||||
onQueueClip: (clip: SceneClip) => Promise<void>;
|
||||
onDequeueClip: (clip: SceneClip) => Promise<void>;
|
||||
/** open the trim dialog for one clip's video */
|
||||
onTrimClip: (clip: SceneClip) => void;
|
||||
/** open the revise dialog for one clip's shown take */
|
||||
onReviseClip: (clip: SceneClip) => void;
|
||||
/** cache-busting version for a job's video preview, bumped after each edit */
|
||||
videoVersionFor: (jobId: string) => number;
|
||||
}
|
||||
|
||||
/** One scene: a timeline of its clips, then every clip's shots in order, numbered straight through. */
|
||||
export function SceneBoard({
|
||||
scene,
|
||||
onOpenJob,
|
||||
onChooseTake,
|
||||
onUseLastFrame,
|
||||
onDetachLastFrame,
|
||||
onQueueScene,
|
||||
onMergeScene,
|
||||
onQueueClip,
|
||||
onDequeueClip,
|
||||
onTrimClip,
|
||||
onReviseClip,
|
||||
videoVersionFor,
|
||||
}: SceneBoardProps) {
|
||||
let shotNumber = 1;
|
||||
const statuses = Object.entries(scene.statusCounts);
|
||||
const anythingToQueue = scene.clips.some((c) => QUEUEABLE_STATUSES.includes(c.job.status));
|
||||
|
||||
// Merging needs every clip's video, so a scene missing one can't pass for the whole thing.
|
||||
const unfinished = scene.clips.filter((c) => c.job.status !== 'completed' || !c.job.output_path).length;
|
||||
const [merging, setMerging] = React.useState(false);
|
||||
async function merge() {
|
||||
setMerging(true);
|
||||
try {
|
||||
await onMergeScene(scene);
|
||||
} finally {
|
||||
setMerging(false);
|
||||
}
|
||||
}
|
||||
|
||||
// The header stays where it is until it reaches the top of the scrolling area,
|
||||
// then sticks there. A 1px sentinel sitting just above it tells us which: once
|
||||
// the sentinel has scrolled out of view, the header is stuck.
|
||||
const sentinelRef = React.useRef<HTMLDivElement>(null);
|
||||
const [stuck, setStuck] = React.useState(false);
|
||||
React.useEffect(() => {
|
||||
const el = sentinelRef.current;
|
||||
if (!el || typeof IntersectionObserver === 'undefined') return;
|
||||
const observer = new IntersectionObserver(([entry]) => setStuck(!entry.isIntersecting));
|
||||
observer.observe(el);
|
||||
return () => observer.disconnect();
|
||||
}, []);
|
||||
|
||||
return (
|
||||
<section aria-label={scene.title} className="flex flex-col gap-4">
|
||||
<div ref={sentinelRef} aria-hidden className="-mb-4 h-px" />
|
||||
<div
|
||||
data-stuck={stuck}
|
||||
className={cn(
|
||||
'sticky top-0 z-30 -mx-3 flex flex-col gap-2 bg-card px-3 pb-3 pt-2 transition-shadow',
|
||||
stuck && 'border-b border-border shadow-md',
|
||||
)}
|
||||
>
|
||||
<div className="flex flex-wrap items-baseline gap-x-4 gap-y-1">
|
||||
<h2 className="text-xl font-semibold text-foreground">{scene.title}</h2>
|
||||
<p className="text-sm text-muted-foreground">
|
||||
{scene.clips.length} clip{scene.clips.length === 1 ? '' : 's'} · {scene.shotCount} shot
|
||||
{scene.shotCount === 1 ? '' : 's'} · about {formatClock(scene.totalSeconds)}
|
||||
</p>
|
||||
<span className="flex flex-wrap gap-1.5">
|
||||
{statuses.map(([status, count]) => (
|
||||
<Badge key={status} variant={BADGE_VARIANTS[status] ?? 'secondary'}>
|
||||
{count} {status}
|
||||
</Badge>
|
||||
))}
|
||||
</span>
|
||||
<span className="ml-auto flex flex-wrap gap-2">
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={merge}
|
||||
disabled={unfinished > 0 || merging}
|
||||
title={
|
||||
unfinished > 0
|
||||
? `${unfinished} clip${unfinished === 1 ? " hasn't" : "s haven't"} finished yet`
|
||||
: 'Join all the clips, in order, into one video'
|
||||
}
|
||||
>
|
||||
{merging ? 'Merging…' : 'Merge scene'}
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
onClick={() => onQueueScene(scene)}
|
||||
disabled={!anythingToQueue}
|
||||
title={anythingToQueue ? 'Queue every clip that has not run yet' : 'Every clip has already run or is queued'}
|
||||
>
|
||||
Queue scene
|
||||
</Button>
|
||||
</span>
|
||||
</div>
|
||||
<Timeline scene={scene} />
|
||||
</div>
|
||||
|
||||
<ol className="flex list-none flex-col gap-3 p-0">
|
||||
{scene.clips.map((clip, index) => {
|
||||
const first = shotNumber;
|
||||
shotNumber += clip.shots.length;
|
||||
return (
|
||||
<ClipRow
|
||||
key={clip.takeKey ?? clip.job.id}
|
||||
clip={clip}
|
||||
previous={scene.clips[index - 1]}
|
||||
firstShotNumber={first}
|
||||
onOpenJob={onOpenJob}
|
||||
onChooseTake={onChooseTake}
|
||||
onUseLastFrame={onUseLastFrame}
|
||||
onDetachLastFrame={onDetachLastFrame}
|
||||
onQueueClip={onQueueClip}
|
||||
onDequeueClip={onDequeueClip}
|
||||
onTrimClip={onTrimClip}
|
||||
onReviseClip={onReviseClip}
|
||||
videoVersionFor={videoVersionFor}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
</ol>
|
||||
</section>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { attachedLastFrames } from '@/lib/continuation';
|
||||
import { planSceneQueue, QUEUEABLE_STATUSES, type SceneQueuePlan } from '@/lib/sceneQueue';
|
||||
import type { Scene, SceneClip } from '@/lib/scenes';
|
||||
|
||||
const plural = (n: number, word: string) => `${n} ${word}${n === 1 ? '' : 's'}`;
|
||||
const clipNumbers = (clips: SceneClip[]) => clips.map((c) => c.position).join(', ');
|
||||
|
||||
export interface SceneQueueDialogProps {
|
||||
scene: Scene | null;
|
||||
onConfirm: (plan: SceneQueuePlan) => Promise<void>;
|
||||
onClose: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Confirm queueing a whole scene: which clips are queued, and whether each
|
||||
* starts from the previous clip's last frame. Nothing is sent until "Queue".
|
||||
*/
|
||||
export function SceneQueueDialog({ scene, onConfirm, onClose }: SceneQueueDialogProps) {
|
||||
// Off by default: the per-clip "Use last frame" boxes are the source of truth, and
|
||||
// this would add links to every clip that doesn't have one.
|
||||
const [chain, setChain] = React.useState(false);
|
||||
const [busy, setBusy] = React.useState(false);
|
||||
|
||||
// Each opening starts from the default choice.
|
||||
React.useEffect(() => {
|
||||
if (scene) setChain(false);
|
||||
}, [scene]);
|
||||
|
||||
const plan = React.useMemo(() => (scene ? planSceneQueue(scene.clips, { chain }) : null), [scene, chain]);
|
||||
const alreadyLinked = React.useMemo(
|
||||
() =>
|
||||
scene
|
||||
? scene.clips.filter(
|
||||
(c) => QUEUEABLE_STATUSES.includes(c.job.status) && attachedLastFrames(c.job.references).length > 0,
|
||||
)
|
||||
: [],
|
||||
[scene],
|
||||
);
|
||||
// Whether the option would do anything is independent of the checkbox.
|
||||
const linkable = React.useMemo(
|
||||
() => (scene ? planSceneQueue(scene.clips, { chain: true }) : null),
|
||||
[scene],
|
||||
);
|
||||
|
||||
async function confirm() {
|
||||
if (!plan) return;
|
||||
setBusy(true);
|
||||
try {
|
||||
await onConfirm(plan);
|
||||
} finally {
|
||||
setBusy(false);
|
||||
}
|
||||
}
|
||||
|
||||
return (
|
||||
<Dialog open={!!scene} onOpenChange={(open) => !open && !busy && onClose()}>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Queue {scene?.title}</DialogTitle>
|
||||
<DialogDescription>
|
||||
{plan && plan.queue.length > 0
|
||||
? `${plural(plan.queue.length, 'clip')} will be queued and run one at a time, in order.`
|
||||
: 'There is nothing in this scene to queue.'}
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
{plan && linkable && (
|
||||
<div className="flex flex-col gap-3 text-sm">
|
||||
{alreadyLinked.length > 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Clip {clipNumbers(alreadyLinked)} already start{alreadyLinked.length === 1 ? 's' : ''} from a previous
|
||||
frame (as you set on the clip{alreadyLinked.length === 1 ? '' : 's'}). Other clips are queued as they are.
|
||||
</p>
|
||||
)}
|
||||
{linkable.chain.length > 0 && (
|
||||
<label className="flex items-start gap-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={chain}
|
||||
onChange={(e) => setChain(e.target.checked)}
|
||||
className="mt-1 size-4"
|
||||
/>
|
||||
<span>
|
||||
Start each clip from the previous clip's last frame
|
||||
<span className="block text-xs text-muted-foreground">
|
||||
Also links {plural(linkable.chain.length, 'clip')} that don't have it (
|
||||
{clipNumbers(linkable.chain.map((c) => c.clip))}), rewriting their prompts. Each waits for the clip
|
||||
before it to finish.
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
)}
|
||||
{chain && plan.cannotChain.length > 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Clip {clipNumbers(plan.cannotChain.map((c) => c.clip))} can't be linked automatically (its prompt
|
||||
isn't in the six-section format) and will start without the previous frame.
|
||||
</p>
|
||||
)}
|
||||
{plan.skipped.length > 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
Skipping{' '}
|
||||
{plan.skipped.map((s) => `clip ${s.clip.position} (${s.reason})`).join(', ')}.
|
||||
</p>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<DialogFooter>
|
||||
<Button type="button" variant="outline" onClick={onClose} disabled={busy}>
|
||||
Cancel
|
||||
</Button>
|
||||
<Button type="button" onClick={confirm} disabled={busy || !plan || plan.queue.length === 0}>
|
||||
{busy ? 'Queueing…' : plan ? `Queue ${plural(plan.queue.length, 'clip')}` : 'Queue'}
|
||||
</Button>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,500 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { toast } from 'sonner';
|
||||
|
||||
import { BeforeAfterGrade } from '@/components/scenes/BeforeAfterGrade';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { Label } from '@/components/ui/label';
|
||||
import { Slider } from '@/components/ui/slider';
|
||||
import { getJobVideoUrl, restoreJobVideo, trimJob } from '@/lib/api';
|
||||
import { BRIGHTNESS_BOUNDS, CONTRAST_BOUNDS, NEUTRAL_GRADE, SATURATION_BOUNDS } from '@/lib/grading';
|
||||
import { clipSeconds, DEFAULT_FPS, formatClock } from '@/lib/scenes';
|
||||
import type { GradeSegment, Job } from '@/lib/types';
|
||||
import { cn } from '@/lib/utils';
|
||||
|
||||
export interface TrimClipDialogProps {
|
||||
/** the clip to edit, or null when closed */
|
||||
clip: Job | null;
|
||||
/**
|
||||
* Cache-busts the preview after an edit (the video keeps the same URL, only
|
||||
* the file changes). Owned by the caller, not this component -- a version
|
||||
* counter kept here gets reset by the very re-render `onChanged` causes
|
||||
* (a fresh `clip` object comes back from the edit), before the browser
|
||||
* ever sees the new URL.
|
||||
*/
|
||||
videoVersion: number;
|
||||
onClose: () => void;
|
||||
/** called with the job as the edit/restore left it, so the caller can refresh */
|
||||
onChanged: (job: Job) => void;
|
||||
}
|
||||
|
||||
/** A section as edited locally -- same shape as GradeSegment plus a stable React key. */
|
||||
interface EditableSection extends GradeSegment {
|
||||
key: string;
|
||||
}
|
||||
|
||||
let sectionSeq = 0;
|
||||
function makeSectionKey(): string {
|
||||
return `section-${Date.now()}-${sectionSeq++}`;
|
||||
}
|
||||
|
||||
const NEUTRAL_SECTION: Omit<EditableSection, 'key'> = { end_seconds: null, ...NEUTRAL_GRADE };
|
||||
|
||||
/** The clip's last-applied sections, or one neutral section covering everything if it was never edited. */
|
||||
function sectionsFromJob(job: Job): EditableSection[] {
|
||||
const segments = job.edit_segments;
|
||||
if (Array.isArray(segments) && segments.length > 0) {
|
||||
return segments.map((s) => ({ ...s, key: makeSectionKey() }));
|
||||
}
|
||||
return [{ ...NEUTRAL_SECTION, key: makeSectionKey() }];
|
||||
}
|
||||
|
||||
// A split closer than this to an existing edge would leave a sliver too thin
|
||||
// to be a meaningful section (and trim.py would likely refuse it outright).
|
||||
const MIN_SPLIT_MARGIN_SECONDS = 0.1;
|
||||
|
||||
/**
|
||||
* Cut a finished clip to a chosen in/out range and grade it section by
|
||||
* section -- like a razor tool: "Split at current time" cuts the clip where
|
||||
* the preview is paused, and each resulting section gets its own brightness,
|
||||
* contrast and saturation. Previous/Next frame step the preview by exactly
|
||||
* one frame (at the clip's own fps), for lining a cut up precisely -- the
|
||||
* native scrub bar alone isn't frame-accurate. The server always keeps the
|
||||
* untouched original and re-renders from it with the full set of values
|
||||
* shown here, so range and sections are applied together and neither is
|
||||
* lost by changing the other --
|
||||
* as long as both are set the way you want them before you click Apply. The
|
||||
* fields keep whatever you last applied while this dialog stays open for the
|
||||
* same clip, so a second, smaller tweak doesn't also reset the range. Opening
|
||||
* it fresh for a clip (or reopening it later) shows exactly what's currently
|
||||
* applied -- the server records the range/sections of the last edit on the
|
||||
* job itself (`edit_*`), so this isn't guessing from the clip's current
|
||||
* length. A never-edited clip shows one neutral section covering its full
|
||||
* length. Restoring clears the fields back to that same neutral state, since
|
||||
* that's what it just did to the video.
|
||||
*/
|
||||
export function TrimClipDialog({ clip, videoVersion, onClose, onChanged }: TrimClipDialogProps) {
|
||||
const duration = clip ? clipSeconds(clip) : 0;
|
||||
const fps = clip?.fps && clip.fps > 0 ? clip.fps : DEFAULT_FPS;
|
||||
const [start, setStart] = React.useState('0');
|
||||
const [end, setEnd] = React.useState('');
|
||||
const [sections, setSections] = React.useState<EditableSection[]>([]);
|
||||
const [previewIndex, setPreviewIndex] = React.useState(0);
|
||||
const [busy, setBusy] = React.useState<'apply' | 'restore' | null>(null);
|
||||
const [displayTime, setDisplayTime] = React.useState(0);
|
||||
|
||||
// Reset the fields only when a *different* clip opens -- onChanged() re-renders
|
||||
// this dialog with a fresh job object for the same edit, which must not reset
|
||||
// what the user is about to apply next.
|
||||
const openClipId = React.useRef<string | null>(null);
|
||||
React.useEffect(() => {
|
||||
if (clip && clip.id !== openClipId.current) {
|
||||
openClipId.current = clip.id;
|
||||
setStart(String(clip.edit_start_seconds ?? 0));
|
||||
setEnd(clip.edit_end_seconds != null ? clip.edit_end_seconds.toFixed(2) : clipSeconds(clip).toFixed(2));
|
||||
setSections(sectionsFromJob(clip));
|
||||
setPreviewIndex(0);
|
||||
} else if (!clip) {
|
||||
openClipId.current = null;
|
||||
}
|
||||
}, [clip]);
|
||||
|
||||
// This dialog is always mounted by the page (it just renders closed when
|
||||
// `clip` is null), so a `useEffect(fn, [])` here would only ever fire once,
|
||||
// the very first time -- before `clip` is ever set and before the <video>
|
||||
// below exists at all -- and never again on later opens. A callback ref
|
||||
// fires on every real mount/unmount of the DOM node itself, however many
|
||||
// times that happens across this component's one long lifetime, which is
|
||||
// what both of these actually need:
|
||||
// - setting the *initial* muted state imperatively, so it doesn't fight the
|
||||
// user's own unmute via the native controls (`muted` as a plain JSX prop
|
||||
// is controlled, and React would re-apply it on every re-render instead)
|
||||
// - keeping the frame-step readout in sync with wherever the video is,
|
||||
// whether that's a step button, a Preview click, or the native scrub bar
|
||||
const videoRef = React.useRef<HTMLVideoElement | null>(null);
|
||||
const setVideoNode = React.useCallback((node: HTMLVideoElement | null) => {
|
||||
videoRef.current = node;
|
||||
if (!node) return;
|
||||
node.muted = true;
|
||||
const onTimeChange = () => setDisplayTime(node.currentTime);
|
||||
node.addEventListener('timeupdate', onTimeChange);
|
||||
node.addEventListener('seeked', onTimeChange);
|
||||
return () => {
|
||||
node.removeEventListener('timeupdate', onTimeChange);
|
||||
node.removeEventListener('seeked', onTimeChange);
|
||||
};
|
||||
}, []);
|
||||
|
||||
if (!clip) {
|
||||
return <Dialog open={false} onOpenChange={() => {}} />;
|
||||
}
|
||||
|
||||
const startSeconds = Number(start);
|
||||
const endSeconds = end.trim() === '' ? undefined : Number(end);
|
||||
|
||||
const rangeError =
|
||||
!Number.isFinite(startSeconds) || startSeconds < 0
|
||||
? 'The start must be 0 or later.'
|
||||
: endSeconds !== undefined && (!Number.isFinite(endSeconds) || endSeconds <= startSeconds)
|
||||
? 'The end must come after the start.'
|
||||
: null;
|
||||
// Color has no equivalent error: the sliders can't produce a value outside their bounds,
|
||||
// and sections are only ever built by split/merge below, which keep them in valid order.
|
||||
const invalid = !!rangeError;
|
||||
|
||||
/** The [start, end) of section `i`, in the kept range's own seconds -- the same timeline the preview plays. */
|
||||
function sectionBounds(i: number): [number, number] {
|
||||
const start = i === 0 ? 0 : (sections[i - 1].end_seconds ?? duration);
|
||||
const end = sections[i].end_seconds ?? duration;
|
||||
return [start, end];
|
||||
}
|
||||
|
||||
function updateSection(i: number, patch: Partial<Omit<EditableSection, 'key' | 'end_seconds'>>) {
|
||||
setSections((prev) => prev.map((s, idx) => (idx === i ? { ...s, ...patch } : s)));
|
||||
}
|
||||
|
||||
/**
|
||||
* Seeks the preview to `target`, clamped to the clip's length. A seek
|
||||
* requested before the video has loaded metadata (readyState is still
|
||||
* HAVE_NOTHING) is unreliable -- browsers commonly drop or silently reset it
|
||||
* once metadata actually arrives -- so this defers to 'loadedmetadata' when
|
||||
* that's the case, instead of jumping to the wrong spot (often 0).
|
||||
*/
|
||||
function seekTo(target: number) {
|
||||
const video = videoRef.current;
|
||||
if (!video) return;
|
||||
const clamped = Math.max(0, Math.min(duration, target));
|
||||
const apply = () => {
|
||||
video.currentTime = clamped;
|
||||
};
|
||||
if (video.readyState >= 1) {
|
||||
apply();
|
||||
} else {
|
||||
video.addEventListener('loadedmetadata', apply, { once: true });
|
||||
}
|
||||
}
|
||||
|
||||
/** Steps the preview by exactly one frame (at the clip's own fps), pausing playback first. */
|
||||
function stepFrame(direction: 1 | -1) {
|
||||
const video = videoRef.current;
|
||||
if (!video) return;
|
||||
video.pause();
|
||||
seekTo(video.currentTime + direction / fps);
|
||||
}
|
||||
|
||||
function splitAtCurrentTime() {
|
||||
const video = videoRef.current;
|
||||
if (!video) return;
|
||||
const t = video.currentTime;
|
||||
for (let i = 0; i < sections.length; i++) {
|
||||
const [sectionStart, sectionEnd] = sectionBounds(i);
|
||||
if (t <= sectionStart + MIN_SPLIT_MARGIN_SECONDS || t >= sectionEnd - MIN_SPLIT_MARGIN_SECONDS) continue;
|
||||
setSections((prev) => {
|
||||
const next = [...prev];
|
||||
next.splice(
|
||||
i,
|
||||
1,
|
||||
{ ...prev[i], end_seconds: t },
|
||||
{ ...NEUTRAL_SECTION, end_seconds: prev[i].end_seconds, key: makeSectionKey() },
|
||||
);
|
||||
return next;
|
||||
});
|
||||
setPreviewIndex(i + 1);
|
||||
return;
|
||||
}
|
||||
toast.error('Pause the preview somewhere inside a section (away from its edges) to split it there.');
|
||||
}
|
||||
|
||||
function mergeWithNext(i: number) {
|
||||
setSections((prev) => {
|
||||
const next = [...prev];
|
||||
next[i] = { ...next[i], end_seconds: next[i + 1].end_seconds };
|
||||
next.splice(i + 1, 1);
|
||||
return next;
|
||||
});
|
||||
setPreviewIndex((p) => Math.min(p, i));
|
||||
}
|
||||
|
||||
/** Seeks the preview to this section's start and previews it; BeforeAfterGrade re-captures on 'seeked'. */
|
||||
function previewSection(i: number) {
|
||||
const [sectionStart] = sectionBounds(i);
|
||||
seekTo(sectionStart);
|
||||
setPreviewIndex(i);
|
||||
}
|
||||
|
||||
/** Keeps the preview panel's selected section in sync with wherever the video actually is. */
|
||||
function handleFrameCaptured(t: number) {
|
||||
for (let i = 0; i < sections.length; i++) {
|
||||
const [, sectionEnd] = sectionBounds(i);
|
||||
if (t < sectionEnd || i === sections.length - 1) {
|
||||
setPreviewIndex(i);
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
async function apply() {
|
||||
if (!clip || invalid) return;
|
||||
setBusy('apply');
|
||||
try {
|
||||
const updated = await trimJob(clip.id, {
|
||||
startSeconds,
|
||||
endSeconds,
|
||||
segments: sections.map(({ key: _key, ...segment }) => segment),
|
||||
});
|
||||
toast.success(`Applied: ${formatClock(startSeconds)}–${endSeconds === undefined ? 'end' : formatClock(endSeconds)}.`);
|
||||
onChanged(updated);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't edit the clip.");
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
}
|
||||
|
||||
async function restore() {
|
||||
if (!clip) return;
|
||||
setBusy('restore');
|
||||
try {
|
||||
const updated = await restoreJobVideo(clip.id);
|
||||
// Restoring doesn't open a *different* clip, so the reset effect above
|
||||
// won't fire on its own -- but the fields have to reflect it too, since
|
||||
// that's exactly what just happened to the video.
|
||||
setStart('0');
|
||||
setEnd(clipSeconds(updated).toFixed(2));
|
||||
setSections([{ ...NEUTRAL_SECTION, key: makeSectionKey() }]);
|
||||
setPreviewIndex(0);
|
||||
toast.success('Restored the original video.');
|
||||
onChanged(updated);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't restore the original video.");
|
||||
} finally {
|
||||
setBusy(null);
|
||||
}
|
||||
}
|
||||
|
||||
const preview = sections[previewIndex] ?? sections[0];
|
||||
|
||||
return (
|
||||
<Dialog open onOpenChange={(open) => !open && !busy && onClose()}>
|
||||
<DialogContent className="max-h-[90vh] max-w-lg overflow-y-auto">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Edit {clip.name?.trim() || clip.id.slice(0, 8)}</DialogTitle>
|
||||
<DialogDescription>
|
||||
Currently {formatClock(duration)}. Editing keeps the untouched original, so this can be adjusted or undone.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<video
|
||||
ref={setVideoNode}
|
||||
src={getJobVideoUrl(clip.id, videoVersion)}
|
||||
aria-label={`Preview of ${clip.name ?? clip.id}`}
|
||||
className="max-h-64 w-full rounded-md border border-border bg-black"
|
||||
controls
|
||||
playsInline
|
||||
preload="metadata"
|
||||
crossOrigin="anonymous"
|
||||
/>
|
||||
|
||||
<div className="flex items-center justify-between gap-2">
|
||||
<div className="flex gap-2">
|
||||
<Button type="button" variant="outline" size="sm" onClick={() => stepFrame(-1)} disabled={!!busy}>
|
||||
◂ Previous frame
|
||||
</Button>
|
||||
<Button type="button" variant="outline" size="sm" onClick={() => stepFrame(1)} disabled={!!busy}>
|
||||
Next frame ▸
|
||||
</Button>
|
||||
</div>
|
||||
<span className="text-xs tabular-nums text-muted-foreground">
|
||||
Frame {Math.round(displayTime * fps)} · {displayTime.toFixed(2)}s
|
||||
</span>
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-2">
|
||||
<p className="text-xs font-medium uppercase tracking-wide text-muted-foreground">Range</p>
|
||||
<div className="flex items-end gap-3">
|
||||
<div className="flex flex-1 flex-col gap-1 text-sm">
|
||||
<Label htmlFor="trim-start">Start (seconds)</Label>
|
||||
<Input
|
||||
id="trim-start"
|
||||
type="number"
|
||||
min={0}
|
||||
step={0.1}
|
||||
value={start}
|
||||
onChange={(e) => setStart(e.target.value)}
|
||||
disabled={!!busy}
|
||||
/>
|
||||
</div>
|
||||
<div className="flex flex-1 flex-col gap-1 text-sm">
|
||||
<Label htmlFor="trim-end">End (seconds, optional)</Label>
|
||||
<Input
|
||||
id="trim-end"
|
||||
type="number"
|
||||
min={0}
|
||||
step={0.1}
|
||||
placeholder="end of clip"
|
||||
value={end}
|
||||
onChange={(e) => setEnd(e.target.value)}
|
||||
disabled={!!busy}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
{rangeError && <p className="text-sm text-destructive">{rangeError}</p>}
|
||||
</div>
|
||||
|
||||
<div className="flex flex-col gap-3">
|
||||
<div className="flex items-center justify-between">
|
||||
<p className="text-xs font-medium uppercase tracking-wide text-muted-foreground">Sections</p>
|
||||
<Button type="button" variant="outline" size="sm" onClick={splitAtCurrentTime} disabled={!!busy}>
|
||||
Split at current time
|
||||
</Button>
|
||||
</div>
|
||||
|
||||
{sections.map((section, i) => {
|
||||
const [sectionStart, sectionEnd] = sectionBounds(i);
|
||||
return (
|
||||
<div
|
||||
key={section.key}
|
||||
className={cn(
|
||||
'flex flex-col gap-2 rounded-md border p-3',
|
||||
i === previewIndex ? 'border-accent-blue' : 'border-border',
|
||||
)}
|
||||
>
|
||||
<div className="flex items-center justify-between">
|
||||
<span className="text-xs font-medium text-muted-foreground">
|
||||
Section {i + 1} · {formatClock(sectionStart)}–{section.end_seconds == null ? 'end' : formatClock(sectionEnd)}
|
||||
</span>
|
||||
<div className="flex gap-3">
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => previewSection(i)}
|
||||
disabled={!!busy}
|
||||
className="text-xs text-accent-blue underline-offset-2 hover:underline disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
Preview
|
||||
</button>
|
||||
{i < sections.length - 1 && (
|
||||
<button
|
||||
type="button"
|
||||
onClick={() => mergeWithNext(i)}
|
||||
disabled={!!busy}
|
||||
className="text-xs text-accent-blue underline-offset-2 hover:underline disabled:cursor-not-allowed disabled:opacity-50"
|
||||
>
|
||||
Merge with next
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
<GradeSlider
|
||||
id={`grade-brightness-${section.key}`}
|
||||
label="Brightness"
|
||||
ariaLabel={`Section ${i + 1} Brightness`}
|
||||
bounds={BRIGHTNESS_BOUNDS}
|
||||
step={5}
|
||||
value={section.brightness}
|
||||
onChange={(v) => updateSection(i, { brightness: v })}
|
||||
format={(v) => String(v)}
|
||||
disabled={!!busy}
|
||||
/>
|
||||
<GradeSlider
|
||||
id={`grade-contrast-${section.key}`}
|
||||
label="Contrast"
|
||||
ariaLabel={`Section ${i + 1} Contrast`}
|
||||
bounds={CONTRAST_BOUNDS}
|
||||
step={0.1}
|
||||
value={section.contrast}
|
||||
onChange={(v) => updateSection(i, { contrast: v })}
|
||||
format={(v) => `${v.toFixed(1)}x`}
|
||||
disabled={!!busy}
|
||||
/>
|
||||
<GradeSlider
|
||||
id={`grade-saturation-${section.key}`}
|
||||
label="Saturation"
|
||||
ariaLabel={`Section ${i + 1} Saturation`}
|
||||
bounds={SATURATION_BOUNDS}
|
||||
step={0.1}
|
||||
value={section.saturation}
|
||||
onChange={(v) => updateSection(i, { saturation: v })}
|
||||
format={(v) => `${v.toFixed(1)}x`}
|
||||
disabled={!!busy}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
})}
|
||||
|
||||
<BeforeAfterGrade
|
||||
videoRef={videoRef}
|
||||
brightness={preview?.brightness ?? NEUTRAL_GRADE.brightness}
|
||||
contrast={preview?.contrast ?? NEUTRAL_GRADE.contrast}
|
||||
saturation={preview?.saturation ?? NEUTRAL_GRADE.saturation}
|
||||
onFrameCaptured={handleFrameCaptured}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<DialogFooter className="justify-between sm:justify-between">
|
||||
<Button type="button" variant="outline" onClick={restore} disabled={!!busy}>
|
||||
{busy === 'restore' ? 'Restoring…' : 'Restore original'}
|
||||
</Button>
|
||||
<div className="flex gap-2">
|
||||
<Button type="button" variant="outline" onClick={onClose} disabled={!!busy}>
|
||||
Close
|
||||
</Button>
|
||||
<Button type="button" onClick={apply} disabled={!!busy || invalid}>
|
||||
{busy === 'apply' ? 'Applying…' : 'Apply'}
|
||||
</Button>
|
||||
</div>
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
|
||||
function GradeSlider({
|
||||
id,
|
||||
label,
|
||||
ariaLabel,
|
||||
bounds,
|
||||
step,
|
||||
value,
|
||||
onChange,
|
||||
format,
|
||||
disabled,
|
||||
}: {
|
||||
id: string;
|
||||
label: string;
|
||||
ariaLabel: string;
|
||||
bounds: readonly [number, number];
|
||||
step: number;
|
||||
value: number;
|
||||
onChange: (v: number) => void;
|
||||
format: (v: number) => string;
|
||||
disabled?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div className="flex flex-col gap-1 text-sm">
|
||||
<div className="flex items-center justify-between">
|
||||
<Label htmlFor={id}>{label}</Label>
|
||||
<span className="text-xs tabular-nums text-muted-foreground">{format(value)}</span>
|
||||
</div>
|
||||
<Slider
|
||||
id={id}
|
||||
min={bounds[0]}
|
||||
max={bounds[1]}
|
||||
step={step}
|
||||
value={[value]}
|
||||
onValueChange={(v) => onChange(v[0])}
|
||||
disabled={disabled}
|
||||
aria-label={ariaLabel}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,389 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { Loader2, Plus, Trash2, Upload } from 'lucide-react';
|
||||
import { toast } from 'sonner';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { NativeSelect } from '@/components/ui/native-select';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import {
|
||||
createJob,
|
||||
getFetchedReferenceUrl,
|
||||
getSceneDraft,
|
||||
startSceneDraft,
|
||||
uploadMedia,
|
||||
type DraftedJob,
|
||||
type MediaType,
|
||||
type SceneDraft,
|
||||
type SceneDraftReference,
|
||||
} from '@/lib/api';
|
||||
import { formatDuration } from '@/lib/jobTiming';
|
||||
|
||||
const POLL_MS = 3000;
|
||||
const SCENE_NAME_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
||||
|
||||
function emptyReference(): SceneDraftReference {
|
||||
return { path: '', media_type: 'image', description: '' };
|
||||
}
|
||||
|
||||
/** Clip length and shot count, read from the drafted job. */
|
||||
function describeJob(job: DraftedJob): string {
|
||||
const seconds = (job.num_frames ?? 0) / (job.fps || 24);
|
||||
const shots = job.prompt.filter((l) => /^\[Shot \d+\]/.test(l)).length;
|
||||
return `${seconds.toFixed(1)}s · ${shots} shot${shots === 1 ? '' : 's'}`;
|
||||
}
|
||||
|
||||
function summaryOf(job: DraftedJob): string {
|
||||
const i = job.prompt.indexOf('summary:');
|
||||
return (i >= 0 ? job.prompt[i + 1] : '')?.replace(/^\[[^\]]*\]\s*/, '') ?? '';
|
||||
}
|
||||
|
||||
export interface WriteSceneDialogProps {
|
||||
open: boolean;
|
||||
onClose: () => void;
|
||||
/** Called after drafts are created as pending jobs. */
|
||||
onCreated: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Draft a scene's jobs from a script with the scene writer agent, review the
|
||||
* drafts, then create them as pending jobs. Nothing is created or started
|
||||
* until "Create jobs".
|
||||
*/
|
||||
export function WriteSceneDialog({ open, onClose, onCreated }: WriteSceneDialogProps) {
|
||||
const [sceneName, setSceneName] = React.useState('');
|
||||
const [script, setScript] = React.useState('');
|
||||
const [references, setReferences] = React.useState<SceneDraftReference[]>([emptyReference()]);
|
||||
const [instructions, setInstructions] = React.useState('');
|
||||
const [findReferences, setFindReferences] = React.useState(false);
|
||||
const [draft, setDraft] = React.useState<SceneDraft | null>(null);
|
||||
const [starting, setStarting] = React.useState(false);
|
||||
const [creating, setCreating] = React.useState(false);
|
||||
const [expanded, setExpanded] = React.useState<string | null>(null);
|
||||
const [now, setNow] = React.useState(() => Date.now());
|
||||
|
||||
const running = draft?.status === 'running';
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!running || !draft) return;
|
||||
const runId = draft.run_id;
|
||||
const poll = setInterval(async () => {
|
||||
try {
|
||||
setDraft(await getSceneDraft(runId));
|
||||
} catch {
|
||||
// Keep the last state; the next poll retries.
|
||||
}
|
||||
}, POLL_MS);
|
||||
const tick = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => {
|
||||
clearInterval(poll);
|
||||
clearInterval(tick);
|
||||
};
|
||||
}, [running, draft?.run_id]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
const usableRefs = references.filter((r) => r.path.trim());
|
||||
const nameValid = SCENE_NAME_RE.test(sceneName);
|
||||
const canStart =
|
||||
nameValid && script.trim().length > 0 && (usableRefs.length > 0 || findReferences) && !running && !starting;
|
||||
|
||||
function updateReference(index: number, patch: Partial<SceneDraftReference>) {
|
||||
setReferences((refs) => refs.map((r, i) => (i === index ? { ...r, ...patch } : r)));
|
||||
}
|
||||
|
||||
async function upload(index: number, file: File) {
|
||||
try {
|
||||
const { path, media_type } = await uploadMedia(file);
|
||||
updateReference(index, { path, media_type });
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : 'Upload failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function start() {
|
||||
setStarting(true);
|
||||
try {
|
||||
setExpanded(null);
|
||||
setDraft(
|
||||
await startSceneDraft({
|
||||
script,
|
||||
scene_name: sceneName,
|
||||
references: usableRefs.map((r) => ({ ...r, path: r.path.trim() })),
|
||||
instructions,
|
||||
find_references: findReferences,
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't start the draft.");
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function createAll() {
|
||||
if (!draft) return;
|
||||
setCreating(true);
|
||||
try {
|
||||
for (const job of draft.jobs) {
|
||||
await createJob({ ...job, prompt: job.prompt.join('\n') });
|
||||
}
|
||||
const n = draft.jobs.length;
|
||||
toast.success(`Created ${n} pending job${n === 1 ? '' : 's'}. Review them, then queue the scene.`);
|
||||
onCreated();
|
||||
setDraft(null);
|
||||
onClose();
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? `${e.message}. Jobs created before the error remain.` : "Couldn't create the jobs.");
|
||||
} finally {
|
||||
setCreating(false);
|
||||
}
|
||||
}
|
||||
|
||||
const elapsed = draft ? ((draft.finished_at ?? now / 1000) - draft.started_at) : 0;
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={(o) => !o && !creating && onClose()}>
|
||||
<DialogContent className="max-h-[90vh] max-w-3xl overflow-y-auto">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Write a scene from a script</DialogTitle>
|
||||
<DialogDescription>
|
||||
The scene writer splits the script into clips and drafts an H3 job for each. Drafts are only created as
|
||||
pending jobs when you press Create.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="flex flex-col gap-4 text-sm">
|
||||
<label className="flex flex-col gap-1">
|
||||
Scene name
|
||||
<Input
|
||||
value={sceneName}
|
||||
onChange={(e) => setSceneName(e.target.value.toLowerCase())}
|
||||
placeholder="whiplash-rehearsal"
|
||||
disabled={running}
|
||||
aria-invalid={sceneName !== '' && !nameValid}
|
||||
/>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Clips are named <code className="font-mono">{sceneName || 'scene'}-clip-01</code>,{' '}
|
||||
<code className="font-mono">-02</code>, … Lowercase letters, digits and dashes.
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<label className="flex flex-col gap-1">
|
||||
Script
|
||||
<Textarea
|
||||
value={script}
|
||||
onChange={(e) => setScript(e.target.value)}
|
||||
rows={10}
|
||||
className="font-mono text-xs"
|
||||
placeholder="Paste the scene: action lines, character names and dialogue."
|
||||
disabled={running}
|
||||
/>
|
||||
</label>
|
||||
|
||||
<fieldset className="flex flex-col gap-2" disabled={running}>
|
||||
<legend className="mb-1">References</legend>
|
||||
{references.map((ref, i) => (
|
||||
<div key={i} className="flex flex-wrap items-center gap-2">
|
||||
<Input
|
||||
value={ref.path}
|
||||
onChange={(e) => updateReference(i, { path: e.target.value })}
|
||||
placeholder="/absolute/path/to/reference.png"
|
||||
className="min-w-[16rem] flex-[2] font-mono text-xs"
|
||||
aria-label={`Reference ${i + 1} path`}
|
||||
/>
|
||||
<NativeSelect
|
||||
value={ref.media_type}
|
||||
onChange={(e) => updateReference(i, { media_type: e.target.value as MediaType })}
|
||||
aria-label={`Reference ${i + 1} type`}
|
||||
className="w-24"
|
||||
>
|
||||
<option value="image">image</option>
|
||||
<option value="audio">audio</option>
|
||||
<option value="video">video</option>
|
||||
</NativeSelect>
|
||||
<Input
|
||||
value={ref.description}
|
||||
onChange={(e) => updateReference(i, { description: e.target.value })}
|
||||
placeholder="What it is, e.g. Fletcher's face / Andrew's voice / the empty rehearsal room"
|
||||
className="min-w-[12rem] flex-[2]"
|
||||
aria-label={`Reference ${i + 1} description`}
|
||||
/>
|
||||
<label className="inline-flex cursor-pointer items-center" title="Upload a file">
|
||||
<Upload className="size-4 text-muted-foreground" aria-hidden />
|
||||
<span className="sr-only">Upload reference {i + 1}</span>
|
||||
<input
|
||||
type="file"
|
||||
accept="image/*,audio/*,video/*"
|
||||
className="sr-only"
|
||||
onChange={(e) => e.target.files?.[0] && upload(i, e.target.files[0])}
|
||||
/>
|
||||
</label>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setReferences((refs) => refs.filter((_, j) => j !== i))}
|
||||
aria-label={`Remove reference ${i + 1}`}
|
||||
disabled={references.length === 1}
|
||||
>
|
||||
<Trash2 className="size-4" aria-hidden />
|
||||
</Button>
|
||||
</div>
|
||||
))}
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="w-fit gap-1.5"
|
||||
onClick={() => setReferences((refs) => [...refs, emptyReference()])}
|
||||
>
|
||||
<Plus className="size-3.5" aria-hidden />
|
||||
Add reference
|
||||
</Button>
|
||||
</fieldset>
|
||||
|
||||
<label className="flex items-start gap-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={findReferences}
|
||||
onChange={(e) => setFindReferences(e.target.checked)}
|
||||
disabled={running}
|
||||
className="mt-1 size-4"
|
||||
/>
|
||||
<span>
|
||||
Find missing references online
|
||||
<span className="block text-xs text-muted-foreground">
|
||||
The writer may download photos from Wikimedia Commons for people, places or objects in the script that
|
||||
none of your references show. You can review each one, with its source and license, before creating
|
||||
jobs.
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<label className="flex flex-col gap-1">
|
||||
Direction (optional)
|
||||
<Textarea
|
||||
value={instructions}
|
||||
onChange={(e) => setInstructions(e.target.value)}
|
||||
rows={3}
|
||||
placeholder="e.g. The first clip is just the band playing; the chair throw comes after."
|
||||
disabled={running}
|
||||
/>
|
||||
</label>
|
||||
|
||||
{draft && (
|
||||
<section aria-live="polite" className="flex flex-col gap-2 rounded-lg border border-border p-3">
|
||||
<div className="flex items-center gap-2">
|
||||
{running && <Loader2 className="size-4 animate-spin text-primary" aria-hidden />}
|
||||
<span className="font-medium">
|
||||
{draft.status === 'running'
|
||||
? 'Drafting…'
|
||||
: draft.status === 'completed'
|
||||
? 'Draft ready'
|
||||
: 'Draft failed'}
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{formatDuration(elapsed)} · {draft.jobs.length} clip{draft.jobs.length === 1 ? '' : 's'} written
|
||||
</span>
|
||||
</div>
|
||||
{draft.error && <p className="text-xs text-destructive">{draft.error}</p>}
|
||||
{draft.status === 'completed' && draft.jobs.length === 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
The writer finished without writing any clips. {draft.final_response ?? ''}
|
||||
</p>
|
||||
)}
|
||||
{draft.fetched_references.length > 0 && (
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<span className="text-xs font-medium">
|
||||
Downloaded references ({draft.fetched_references.length}): check these before creating jobs
|
||||
</span>
|
||||
<ul className="grid grid-cols-2 gap-2 sm:grid-cols-4">
|
||||
{draft.fetched_references.map((ref) => {
|
||||
const used = draft.jobs.some((j) => j.references?.some((r) => r.source === ref.path));
|
||||
return (
|
||||
<li key={ref.path} className="flex flex-col gap-1 rounded-md bg-secondary/40 p-1.5 text-[11px]">
|
||||
{/* eslint-disable-next-line @next/next/no-img-element -- served by the studio API, not Next */}
|
||||
<img
|
||||
src={getFetchedReferenceUrl(draft.run_id, ref)}
|
||||
alt={`${ref.subject}: ${ref.title}`}
|
||||
className="aspect-square w-full rounded object-cover"
|
||||
/>
|
||||
<span className="font-medium">
|
||||
{ref.subject}
|
||||
{used ? ' · used' : ''}
|
||||
</span>
|
||||
<a
|
||||
href={ref.page}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="truncate text-muted-foreground underline"
|
||||
title={ref.title}
|
||||
>
|
||||
{ref.title}
|
||||
</a>
|
||||
<span className="truncate text-muted-foreground" title={`${ref.license} · ${ref.artist}`}>
|
||||
{ref.license} · {ref.artist}
|
||||
</span>
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
<ol className="flex flex-col gap-1.5">
|
||||
{draft.jobs.map((job) => (
|
||||
<li key={job.name} className="rounded-md bg-secondary/40 p-2">
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full flex-wrap items-baseline gap-2 text-left"
|
||||
onClick={() => setExpanded((n) => (n === job.name ? null : job.name ?? null))}
|
||||
aria-expanded={expanded === job.name}
|
||||
>
|
||||
<span className="font-mono text-xs font-semibold">{job.name}</span>
|
||||
<span className="text-xs text-muted-foreground">{describeJob(job)}</span>
|
||||
<span className="w-full truncate text-xs text-muted-foreground">{summaryOf(job)}</span>
|
||||
</button>
|
||||
{expanded === job.name && (
|
||||
<pre className="mt-2 max-h-72 overflow-auto whitespace-pre-wrap rounded bg-background p-2 text-[11px]">
|
||||
{job.prompt.join('\n')}
|
||||
</pre>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<DialogFooter>
|
||||
<Button type="button" variant="outline" onClick={onClose} disabled={creating}>
|
||||
{running ? 'Hide' : 'Cancel'}
|
||||
</Button>
|
||||
{draft?.status === 'completed' && draft.jobs.length > 0 ? (
|
||||
<>
|
||||
<Button type="button" variant="outline" onClick={start} disabled={!canStart || creating}>
|
||||
Redraft
|
||||
</Button>
|
||||
<Button type="button" onClick={createAll} disabled={creating}>
|
||||
{creating ? 'Creating…' : `Create ${draft.jobs.length} pending job${draft.jobs.length === 1 ? '' : 's'}`}
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<Button type="button" onClick={start} disabled={!canStart}>
|
||||
{starting ? 'Starting…' : running ? 'Drafting…' : 'Draft clips'}
|
||||
</Button>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -13,6 +13,7 @@ const TAB_TITLES: Record<string, string> = {
|
||||
'/distillation': 'Jobs',
|
||||
'/datasets': 'Datasets',
|
||||
'/gallery': 'Gallery',
|
||||
'/scenes': 'Scenes',
|
||||
'/gpus': 'GPUs',
|
||||
'/settings': 'Settings',
|
||||
};
|
||||
|
||||
@@ -150,6 +150,14 @@ export default function PrimarySidebar({
|
||||
>
|
||||
Datasets
|
||||
</Link>
|
||||
<Link
|
||||
href="/scenes"
|
||||
aria-current={pathname === '/scenes' ? 'page' : undefined}
|
||||
onClick={onMobileClose}
|
||||
className={cn(TAB_BASE, pathname === '/scenes' && TAB_ACTIVE)}
|
||||
>
|
||||
Scenes
|
||||
</Link>
|
||||
<Link
|
||||
href="/gallery"
|
||||
aria-current={pathname === '/gallery' ? 'page' : undefined}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { render, screen } from '@testing-library/react';
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { Dialog, DialogContent, DialogDescription, DialogTitle } from '@/components/ui/dialog';
|
||||
|
||||
// The app header is `fixed` at z-[100] (components/shell/Header.tsx).
|
||||
const HEADER_Z = 100;
|
||||
|
||||
function zOf(el: Element | null): number {
|
||||
const m = el?.className.match(/\bz-\[(\d+)\]/);
|
||||
return m ? Number(m[1]) : 0;
|
||||
}
|
||||
|
||||
describe('Dialog stacking', () => {
|
||||
it('renders its overlay and content above the fixed app header', () => {
|
||||
render(
|
||||
<Dialog open>
|
||||
<DialogContent>
|
||||
<DialogTitle>Tall dialog</DialogTitle>
|
||||
<DialogDescription>Body</DialogDescription>
|
||||
</DialogContent>
|
||||
</Dialog>,
|
||||
);
|
||||
const content = screen.getByRole('dialog');
|
||||
const overlay = document.querySelector('[data-state="open"].bg-black\\/70');
|
||||
|
||||
expect(zOf(content)).toBeGreaterThan(HEADER_Z);
|
||||
expect(zOf(overlay)).toBeGreaterThan(HEADER_Z);
|
||||
});
|
||||
});
|
||||
@@ -11,6 +11,8 @@ const DialogTrigger = DialogPrimitive.Trigger;
|
||||
const DialogPortal = DialogPrimitive.Portal;
|
||||
const DialogClose = DialogPrimitive.Close;
|
||||
|
||||
// Dialogs sit above the app header (z-[100], fixed). Below it, a tall dialog's
|
||||
// top edge slides under the header and, being fixed itself, can't be scrolled clear.
|
||||
const DialogOverlay = React.forwardRef<
|
||||
React.ElementRef<typeof DialogPrimitive.Overlay>,
|
||||
React.ComponentPropsWithoutRef<typeof DialogPrimitive.Overlay>
|
||||
@@ -18,7 +20,7 @@ const DialogOverlay = React.forwardRef<
|
||||
<DialogPrimitive.Overlay
|
||||
ref={ref}
|
||||
className={cn(
|
||||
'fixed inset-0 z-50 bg-black/70 backdrop-blur-sm data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0',
|
||||
'fixed inset-0 z-[110] bg-black/70 backdrop-blur-sm data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0',
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
@@ -36,7 +38,7 @@ const DialogContent = React.forwardRef<
|
||||
ref={ref}
|
||||
data-modal=""
|
||||
className={cn(
|
||||
'fixed left-1/2 top-1/2 z-50 grid w-full max-w-lg -translate-x-1/2 -translate-y-1/2 gap-4 rounded-2xl border border-border bg-card p-6 text-card-foreground shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0',
|
||||
'fixed left-1/2 top-1/2 z-[110] grid w-full max-w-lg -translate-x-1/2 -translate-y-1/2 gap-4 rounded-2xl border border-border bg-card p-6 text-card-foreground shadow-lg duration-200 data-[state=open]:animate-in data-[state=closed]:animate-out data-[state=closed]:fade-out-0 data-[state=open]:fade-in-0',
|
||||
className,
|
||||
)}
|
||||
{...props}
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
import { computeElapsed } from '@/lib/jobTiming';
|
||||
import type { Job } from '@/lib/types';
|
||||
|
||||
/** A job's formatted run time, ticking every second while it runs. */
|
||||
export function useJobElapsed(job: Job): string | null {
|
||||
const [currentTime, setCurrentTime] = React.useState(() => Date.now());
|
||||
|
||||
React.useEffect(() => {
|
||||
if (job.status !== 'running' || !job.started_at) return;
|
||||
const interval = setInterval(() => setCurrentTime(Date.now()), 1000);
|
||||
return () => clearInterval(interval);
|
||||
}, [job.status, job.started_at]);
|
||||
|
||||
return computeElapsed(job, currentTime);
|
||||
}
|
||||
@@ -0,0 +1,140 @@
|
||||
import { act, renderHook } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { COALESCE_MS, useUndoableState } from './useUndoableState';
|
||||
|
||||
describe('useUndoableState', () => {
|
||||
let now = 1_000_000;
|
||||
beforeEach(() => {
|
||||
now = 1_000_000;
|
||||
vi.spyOn(Date, 'now').mockImplementation(() => now);
|
||||
});
|
||||
afterEach(() => vi.restoreAllMocks());
|
||||
|
||||
const tick = (ms: number) => {
|
||||
now += ms;
|
||||
};
|
||||
|
||||
it('makes each unkeyed change its own step', () => {
|
||||
const { result } = renderHook(() => useUndoableState({ n: 0 }));
|
||||
act(() => result.current.set({ n: 1 }));
|
||||
act(() => result.current.set({ n: 2 }));
|
||||
expect(result.current.state.n).toBe(2);
|
||||
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state.n).toBe(1);
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state.n).toBe(0);
|
||||
expect(result.current.canUndo).toBe(false);
|
||||
});
|
||||
|
||||
it('merges same-key edits made close together into one step', () => {
|
||||
const { result } = renderHook(() => useUndoableState('a'));
|
||||
act(() => result.current.set('ab', 'text'));
|
||||
tick(100);
|
||||
act(() => result.current.set('abc', 'text'));
|
||||
tick(100);
|
||||
act(() => result.current.set('abcd', 'text'));
|
||||
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state).toBe('a');
|
||||
expect(result.current.canUndo).toBe(false);
|
||||
});
|
||||
|
||||
it('starts a new step after a pause, or when the key changes', () => {
|
||||
const { result } = renderHook(() => useUndoableState('a'));
|
||||
act(() => result.current.set('ab', 'text'));
|
||||
tick(COALESCE_MS + 1);
|
||||
act(() => result.current.set('abc', 'text'));
|
||||
act(() => result.current.set('abcX', 'other'));
|
||||
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state).toBe('abc');
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state).toBe('ab');
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state).toBe('a');
|
||||
});
|
||||
|
||||
it('never merges an unkeyed change into typing, or typing into it', () => {
|
||||
const { result } = renderHook(() => useUndoableState('a'));
|
||||
act(() => result.current.set('ab', 'text'));
|
||||
act(() => result.current.set('AB'));
|
||||
act(() => result.current.set('ABc', 'text'));
|
||||
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state).toBe('AB');
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state).toBe('ab');
|
||||
});
|
||||
|
||||
it('typing after an undo begins a fresh step instead of merging into the undone one', () => {
|
||||
const { result } = renderHook(() => useUndoableState('a'));
|
||||
act(() => result.current.set('ab', 'text'));
|
||||
act(() => void result.current.undo());
|
||||
act(() => result.current.set('ax', 'text'));
|
||||
expect(result.current.canUndo).toBe(true);
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state).toBe('a');
|
||||
});
|
||||
|
||||
it('redoes, and a new change clears what could be redone', () => {
|
||||
const { result } = renderHook(() => useUndoableState(0));
|
||||
act(() => result.current.set(1));
|
||||
act(() => result.current.set(2));
|
||||
act(() => void result.current.undo());
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.canRedo).toBe(true);
|
||||
|
||||
act(() => void result.current.redo());
|
||||
expect(result.current.state).toBe(1);
|
||||
act(() => result.current.set(9));
|
||||
expect(result.current.canRedo).toBe(false);
|
||||
expect(result.current.redo()).toBe(false);
|
||||
});
|
||||
|
||||
it('returns false when there is nothing to undo or redo', () => {
|
||||
const { result } = renderHook(() => useUndoableState(0));
|
||||
expect(result.current.undo()).toBe(false);
|
||||
expect(result.current.redo()).toBe(false);
|
||||
});
|
||||
|
||||
it('chains functional updates made in the same event', () => {
|
||||
const { result } = renderHook(() => useUndoableState(0));
|
||||
act(() => {
|
||||
result.current.set((n) => n + 1);
|
||||
result.current.set((n) => n + 1);
|
||||
});
|
||||
expect(result.current.state).toBe(2);
|
||||
act(() => void result.current.undo());
|
||||
expect(result.current.state).toBe(1);
|
||||
});
|
||||
|
||||
it('ignores a set that produces the same value', () => {
|
||||
const { result } = renderHook(() => useUndoableState('x'));
|
||||
act(() => result.current.set('x'));
|
||||
expect(result.current.canUndo).toBe(false);
|
||||
});
|
||||
|
||||
it('reset replaces the value and drops all history', () => {
|
||||
const { result } = renderHook(() => useUndoableState(0));
|
||||
act(() => result.current.set(1));
|
||||
act(() => void result.current.undo());
|
||||
act(() => result.current.reset(7));
|
||||
expect(result.current.state).toBe(7);
|
||||
expect(result.current.canUndo).toBe(false);
|
||||
expect(result.current.canRedo).toBe(false);
|
||||
});
|
||||
|
||||
it('keeps at most 100 steps', () => {
|
||||
const { result } = renderHook(() => useUndoableState(0));
|
||||
for (let i = 1; i <= 130; i += 1) act(() => result.current.set(i));
|
||||
let undone = 0;
|
||||
while (result.current.canUndo) {
|
||||
act(() => void result.current.undo());
|
||||
undone += 1;
|
||||
}
|
||||
expect(undone).toBe(100);
|
||||
expect(result.current.state).toBe(30);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,86 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
const MAX_HISTORY = 100;
|
||||
// Edits made under the same key, this close together, merge into one undo step.
|
||||
export const COALESCE_MS = 500;
|
||||
|
||||
interface Snapshot<T> {
|
||||
present: T;
|
||||
past: T[];
|
||||
future: T[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Undo/redo over a whole state value. Every `set` is one undo step, except
|
||||
* that consecutive sets with the same `key` inside COALESCE_MS merge -- pass a
|
||||
* key for typing so a burst is one step, and none for discrete changes (moves,
|
||||
* adds, selects) so each is its own. Values are treated as immutable, so
|
||||
* snapshots share structure and are cheap to keep.
|
||||
*/
|
||||
export function useUndoableState<T>(initial: T) {
|
||||
const [snapshot, setSnapshot] = React.useState<Snapshot<T>>({ present: initial, past: [], future: [] });
|
||||
// Mirrors the snapshot synchronously so several sets in one event chain correctly.
|
||||
const ref = React.useRef(snapshot);
|
||||
const last = React.useRef<{ key: string | null; at: number }>({ key: null, at: 0 });
|
||||
|
||||
const apply = React.useCallback((next: Snapshot<T>) => {
|
||||
ref.current = next;
|
||||
setSnapshot(next);
|
||||
}, []);
|
||||
|
||||
const set = React.useCallback(
|
||||
(update: T | ((prev: T) => T), key: string | null = null) => {
|
||||
const cur = ref.current;
|
||||
const next = typeof update === 'function' ? (update as (prev: T) => T)(cur.present) : update;
|
||||
if (Object.is(next, cur.present)) return;
|
||||
|
||||
const now = Date.now();
|
||||
const merges = key !== null && last.current.key === key && now - last.current.at < COALESCE_MS;
|
||||
last.current = { key, at: now };
|
||||
|
||||
if (merges) {
|
||||
apply({ ...cur, present: next });
|
||||
} else {
|
||||
apply({ present: next, past: [...cur.past.slice(-(MAX_HISTORY - 1)), cur.present], future: [] });
|
||||
}
|
||||
},
|
||||
[apply],
|
||||
);
|
||||
|
||||
const undo = React.useCallback(() => {
|
||||
const cur = ref.current;
|
||||
if (cur.past.length === 0) return false;
|
||||
last.current = { key: null, at: 0 };
|
||||
apply({ present: cur.past[cur.past.length - 1], past: cur.past.slice(0, -1), future: [cur.present, ...cur.future] });
|
||||
return true;
|
||||
}, [apply]);
|
||||
|
||||
const redo = React.useCallback(() => {
|
||||
const cur = ref.current;
|
||||
if (cur.future.length === 0) return false;
|
||||
last.current = { key: null, at: 0 };
|
||||
apply({ present: cur.future[0], past: [...cur.past, cur.present], future: cur.future.slice(1) });
|
||||
return true;
|
||||
}, [apply]);
|
||||
|
||||
/** Replace the value and forget all history (e.g. when an editor reopens). */
|
||||
const reset = React.useCallback(
|
||||
(value: T) => {
|
||||
last.current = { key: null, at: 0 };
|
||||
apply({ present: value, past: [], future: [] });
|
||||
},
|
||||
[apply],
|
||||
);
|
||||
|
||||
return {
|
||||
state: snapshot.present,
|
||||
set,
|
||||
undo,
|
||||
redo,
|
||||
reset,
|
||||
canUndo: snapshot.past.length > 0,
|
||||
canRedo: snapshot.future.length > 0,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,84 @@
|
||||
import { act, renderHook } from '@testing-library/react';
|
||||
import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
|
||||
|
||||
import { useUndoableText } from './useUndoableText';
|
||||
|
||||
describe('useUndoableText', () => {
|
||||
beforeEach(() => vi.useFakeTimers());
|
||||
afterEach(() => vi.useRealTimers());
|
||||
|
||||
it('coalesces a typing burst into one undo step', () => {
|
||||
const { result } = renderHook(() => useUndoableText('a'));
|
||||
act(() => {
|
||||
result.current.type('ab');
|
||||
result.current.type('abc');
|
||||
});
|
||||
act(() => {
|
||||
vi.advanceTimersByTime(600);
|
||||
});
|
||||
expect(result.current.canUndo).toBe(true);
|
||||
|
||||
let restored: string | null = null;
|
||||
act(() => {
|
||||
restored = result.current.undo();
|
||||
});
|
||||
expect(restored).toBe('a');
|
||||
expect(result.current.text).toBe('a');
|
||||
expect(result.current.canUndo).toBe(false);
|
||||
});
|
||||
|
||||
it('gives a committed edit its own step and supports redo', () => {
|
||||
const { result } = renderHook(() => useUndoableText('hi'));
|
||||
act(() => result.current.commit('hi <pause>'));
|
||||
expect(result.current.text).toBe('hi <pause>');
|
||||
|
||||
act(() => {
|
||||
result.current.undo();
|
||||
});
|
||||
expect(result.current.text).toBe('hi');
|
||||
expect(result.current.canRedo).toBe(true);
|
||||
|
||||
let redone: string | null = null;
|
||||
act(() => {
|
||||
redone = result.current.redo();
|
||||
});
|
||||
expect(redone).toBe('hi <pause>');
|
||||
expect(result.current.text).toBe('hi <pause>');
|
||||
});
|
||||
|
||||
it('flushes pending typing before a commit so undo steps stay separate', () => {
|
||||
const { result } = renderHook(() => useUndoableText(''));
|
||||
act(() => result.current.type('hello'));
|
||||
act(() => result.current.commit('hello <pause>'));
|
||||
|
||||
act(() => {
|
||||
result.current.undo();
|
||||
});
|
||||
expect(result.current.text).toBe('hello');
|
||||
act(() => {
|
||||
result.current.undo();
|
||||
});
|
||||
expect(result.current.text).toBe('');
|
||||
});
|
||||
|
||||
it('returns null when there is nothing to undo or redo', () => {
|
||||
const { result } = renderHook(() => useUndoableText('x'));
|
||||
let undone: string | null = 'sentinel';
|
||||
let redone: string | null = 'sentinel';
|
||||
act(() => {
|
||||
undone = result.current.undo();
|
||||
redone = result.current.redo();
|
||||
});
|
||||
expect(undone).toBeNull();
|
||||
expect(redone).toBeNull();
|
||||
});
|
||||
|
||||
it('reset replaces the text and drops history', () => {
|
||||
const { result } = renderHook(() => useUndoableText('a'));
|
||||
act(() => result.current.commit('b'));
|
||||
act(() => result.current.reset('fresh'));
|
||||
expect(result.current.text).toBe('fresh');
|
||||
expect(result.current.canUndo).toBe(false);
|
||||
expect(result.current.canRedo).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,109 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
|
||||
const MAX_HISTORY = 100;
|
||||
// How long a pause in typing has to be before it becomes its own undo step.
|
||||
// Keeps undo from removing text one keystroke at a time while still giving
|
||||
// each tag/preset insertion (the actual ask) a guaranteed, immediate step.
|
||||
const TYPING_COALESCE_MS = 500;
|
||||
|
||||
/**
|
||||
* Text state with undo/redo. Typing is coalesced into bursts; discrete edits
|
||||
* (`commit`, e.g. a tag insertion) always get their own step. `textRef`
|
||||
* mirrors `text` synchronously because history bookkeeping needs the true
|
||||
* current value the instant an edit happens, not whatever the last render
|
||||
* closed over.
|
||||
*/
|
||||
export function useUndoableText(initial: string) {
|
||||
const [text, setTextState] = React.useState(initial);
|
||||
const textRef = React.useRef(initial);
|
||||
const [past, setPast] = React.useState<string[]>([]);
|
||||
const [future, setFuture] = React.useState<string[]>([]);
|
||||
// Start of the in-progress typing burst that hasn't become a history step
|
||||
// yet; flushed into `past` once typing pauses or a discrete edit happens.
|
||||
const typingBaseRef = React.useRef<string | null>(null);
|
||||
const typingTimerRef = React.useRef<ReturnType<typeof setTimeout> | null>(null);
|
||||
|
||||
const setText = React.useCallback((next: string) => {
|
||||
textRef.current = next;
|
||||
setTextState(next);
|
||||
}, []);
|
||||
|
||||
const pushHistory = React.useCallback((previous: string) => {
|
||||
setPast((p) => (p.length >= MAX_HISTORY ? [...p.slice(1), previous] : [...p, previous]));
|
||||
setFuture([]);
|
||||
}, []);
|
||||
|
||||
const clearTypingTimer = React.useCallback(() => {
|
||||
if (typingTimerRef.current) {
|
||||
clearTimeout(typingTimerRef.current);
|
||||
typingTimerRef.current = null;
|
||||
}
|
||||
}, []);
|
||||
|
||||
const flushTypingCheckpoint = React.useCallback(() => {
|
||||
clearTypingTimer();
|
||||
if (typingBaseRef.current != null && typingBaseRef.current !== textRef.current) {
|
||||
pushHistory(typingBaseRef.current);
|
||||
}
|
||||
typingBaseRef.current = null;
|
||||
}, [clearTypingTimer, pushHistory]);
|
||||
|
||||
/** Free typing: coalesced into one undo step per burst. */
|
||||
const type = React.useCallback(
|
||||
(next: string) => {
|
||||
if (typingBaseRef.current == null) typingBaseRef.current = textRef.current;
|
||||
setText(next);
|
||||
clearTypingTimer();
|
||||
typingTimerRef.current = setTimeout(flushTypingCheckpoint, TYPING_COALESCE_MS);
|
||||
},
|
||||
[setText, clearTypingTimer, flushTypingCheckpoint],
|
||||
);
|
||||
|
||||
/** Discrete edit (tag/preset insertion): always its own undo step. */
|
||||
const commit = React.useCallback(
|
||||
(next: string) => {
|
||||
flushTypingCheckpoint();
|
||||
pushHistory(textRef.current);
|
||||
setText(next);
|
||||
},
|
||||
[flushTypingCheckpoint, pushHistory, setText],
|
||||
);
|
||||
|
||||
/** Returns the restored text (so the caller can place the caret), or null if nothing to undo. */
|
||||
function undo(): string | null {
|
||||
flushTypingCheckpoint();
|
||||
if (past.length === 0) return null;
|
||||
const prev = past[past.length - 1];
|
||||
setPast(past.slice(0, -1));
|
||||
setFuture([...future, textRef.current]);
|
||||
setText(prev);
|
||||
return prev;
|
||||
}
|
||||
|
||||
function redo(): string | null {
|
||||
if (future.length === 0) return null;
|
||||
const next = future[future.length - 1];
|
||||
setFuture(future.slice(0, -1));
|
||||
setPast([...past, textRef.current]);
|
||||
setText(next);
|
||||
return next;
|
||||
}
|
||||
|
||||
/** Replace the text and drop all history (e.g. when an editor is reopened). */
|
||||
const reset = React.useCallback(
|
||||
(value: string) => {
|
||||
setText(value);
|
||||
setPast([]);
|
||||
setFuture([]);
|
||||
typingBaseRef.current = null;
|
||||
clearTypingTimer();
|
||||
},
|
||||
[setText, clearTypingTimer],
|
||||
);
|
||||
|
||||
React.useEffect(() => clearTypingTimer, [clearTypingTimer]);
|
||||
|
||||
return { text, type, commit, undo, redo, reset, canUndo: past.length > 0, canRedo: future.length > 0 };
|
||||
}
|
||||
@@ -5,7 +5,7 @@ import {
|
||||
loadDefaultOptions,
|
||||
type DefaultOptions,
|
||||
} from "./defaultOptions";
|
||||
import type { Job, JobType } from "./types";
|
||||
import type { GradeSegment, Job, JobType } from "./types";
|
||||
|
||||
const DEFAULT_API_BASE_URL = "http://localhost:8189/api";
|
||||
|
||||
@@ -50,8 +50,20 @@ function stripTrailingSlash(url: string): string {
|
||||
}
|
||||
|
||||
/** Full URL for streaming a job's video/image output (for <video> or <img> src). */
|
||||
export function getJobVideoUrl(jobId: string): string {
|
||||
return `${getApiBaseUrl()}/jobs/${jobId}/video`;
|
||||
/**
|
||||
* `version` busts the browser's cache after editing a job's video in place
|
||||
* (Trim/Edit video): the URL is otherwise identical before and after an edit,
|
||||
* even though the file on disk changed, so without this the same cached
|
||||
* response keeps showing. Omit it where staleness doesn't matter.
|
||||
*/
|
||||
export function getJobVideoUrl(jobId: string, version?: number): string {
|
||||
const base = `${getApiBaseUrl()}/jobs/${jobId}/video`;
|
||||
return version ? `${base}?v=${version}` : base;
|
||||
}
|
||||
|
||||
/** One of a job's image references, served only when the job lists it. */
|
||||
export function getJobReferenceUrl(jobId: string, source: string): string {
|
||||
return `${getApiBaseUrl()}/jobs/${jobId}/reference?source=${encodeURIComponent(source)}`;
|
||||
}
|
||||
|
||||
export interface CreateJobRequest {
|
||||
@@ -62,6 +74,8 @@ export interface CreateJobRequest {
|
||||
workload_type?: string;
|
||||
job_type?: JobType;
|
||||
image_path?: string;
|
||||
/** Ref2VA reference media, numbered by type in this order (<Picture 1>, <Audio 1>, ...). */
|
||||
references?: { source: string; media_type: MediaType }[];
|
||||
data_path?: string;
|
||||
max_train_steps?: number;
|
||||
train_batch_size?: number;
|
||||
@@ -194,6 +208,84 @@ export async function updateJob(
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the server to save the last frame of a finished clip and return its path,
|
||||
* ready to attach to another job as an image reference.
|
||||
*/
|
||||
export async function extractLastFrame(jobId: string): Promise<{ path: string; media_type: MediaType }> {
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
const response = await fetch(`${baseApiUrl}/jobs/${jobId}/last-frame`, { method: "POST" });
|
||||
if (!response.ok) {
|
||||
const err = await response.json().catch(() => ({ detail: "Could not get the last frame" }));
|
||||
throw new Error(err.detail || "Could not get the last frame");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export interface EditVideoParams {
|
||||
startSeconds: number;
|
||||
/** omitted (or undefined) keeps to the end of the video */
|
||||
endSeconds?: number;
|
||||
/**
|
||||
* Ordered, contiguous razor-cut sections covering the whole kept range --
|
||||
* each section's `end_seconds` is relative to that kept range, not the
|
||||
* original file. Omitted (or empty) means one neutral section, i.e. a
|
||||
* plain range trim with no grading.
|
||||
*/
|
||||
segments?: GradeSegment[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Cut a completed job's video to a range and/or grade it section by section,
|
||||
* in place. The untouched original is kept server-side, so this always
|
||||
* renders fresh from it with the *given* parameters -- pass every value you
|
||||
* want kept, not just the one you changed, or the rest reset to their neutral
|
||||
* defaults. Can be undone with restoreJobVideo.
|
||||
*/
|
||||
export async function trimJob(jobId: string, params: EditVideoParams): Promise<Job> {
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
const response = await fetch(`${baseApiUrl}/jobs/${jobId}/trim`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({
|
||||
start_seconds: params.startSeconds,
|
||||
end_seconds: params.endSeconds ?? null,
|
||||
segments: params.segments ?? [],
|
||||
}),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const err = await response.json().catch(() => ({ detail: "Could not edit the clip" }));
|
||||
throw new Error(err.detail || "Could not edit the clip");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/** Undo every trim and color adjustment on a job's video, back to what it originally generated. */
|
||||
export async function restoreJobVideo(jobId: string): Promise<Job> {
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
const response = await fetch(`${baseApiUrl}/jobs/${jobId}/restore-video`, { method: "POST" });
|
||||
if (!response.ok) {
|
||||
const err = await response.json().catch(() => ({ detail: "Could not restore the original video" }));
|
||||
throw new Error(err.detail || "Could not restore the original video");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Ask the server to save the last second or so of a finished clip and return its
|
||||
* path, ready to attach to another job as a video reference -- so the next
|
||||
* clip continues with real motion instead of guessing it from a single frame.
|
||||
*/
|
||||
export async function extractLastClip(jobId: string): Promise<{ path: string; media_type: MediaType }> {
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
const response = await fetch(`${baseApiUrl}/jobs/${jobId}/last-clip`, { method: "POST" });
|
||||
if (!response.ok) {
|
||||
const err = await response.json().catch(() => ({ detail: "Could not get the end of the clip" }));
|
||||
throw new Error(err.detail || "Could not get the end of the clip");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function uploadMedia(
|
||||
file: File,
|
||||
): Promise<{ path: string; media_type: MediaType }> {
|
||||
@@ -313,6 +405,80 @@ export async function startJob(id: string): Promise<Job> {
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Queue jobs, in the order given, to run one after another as the server has room.
|
||||
* A job that starts from another job's last frame waits for that job to finish.
|
||||
* All-or-nothing: if any job can't be queued, none are.
|
||||
*/
|
||||
export async function queueJobs(ids: string[]): Promise<Job[]> {
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
const response = await fetch(`${baseApiUrl}/jobs/queue`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ job_ids: ids }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response
|
||||
.json()
|
||||
.catch(() => ({ detail: "Failed to queue jobs" }));
|
||||
throw new Error(error.detail || "Failed to queue jobs");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/** Take a queued job out of the queue; it goes back to pending. */
|
||||
export async function dequeueJob(id: string): Promise<Job> {
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
const response = await fetch(`${baseApiUrl}/jobs/${id}/dequeue`, {
|
||||
method: "POST",
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response
|
||||
.json()
|
||||
.catch(() => ({ detail: "Failed to remove job from the queue" }));
|
||||
throw new Error(error.detail || "Failed to remove job from the queue");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export interface MergedScene {
|
||||
/** file name on the server; pass to getMergedVideoUrl / downloadMergedVideo */
|
||||
filename: string;
|
||||
clips: number;
|
||||
/** length of the merged video */
|
||||
seconds: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Join the finished clips of a scene, in the order given, into one video. Fails,
|
||||
* making nothing, if any clip hasn't finished.
|
||||
*/
|
||||
export async function mergeScene(jobIds: string[], name: string): Promise<MergedScene> {
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
const response = await fetch(`${baseApiUrl}/scenes/merge`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify({ job_ids: jobIds, name }),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({ detail: "Failed to merge the scene" }));
|
||||
throw new Error(error.detail || "Failed to merge the scene");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export function getMergedVideoUrl(filename: string): string {
|
||||
return `${getApiBaseUrl()}/merged/${encodeURIComponent(filename)}`;
|
||||
}
|
||||
|
||||
export async function downloadMergedVideo(filename: string): Promise<Blob> {
|
||||
const response = await fetch(getMergedVideoUrl(filename));
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to download the merged video");
|
||||
}
|
||||
return response.blob();
|
||||
}
|
||||
|
||||
export async function stopJob(id: string): Promise<Job> {
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
const response = await fetch(`${baseApiUrl}/jobs/${id}/stop`, {
|
||||
@@ -479,3 +645,118 @@ export function getDatasetMediaUrl(
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
return `${baseApiUrl}/datasets/${datasetId}/media/${encodeURIComponent(fileName)}`;
|
||||
}
|
||||
|
||||
// MARK: - Scene writer
|
||||
|
||||
export interface SceneDraftReference {
|
||||
path: string;
|
||||
media_type: MediaType;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface SceneDraftRequest {
|
||||
script: string;
|
||||
/** Lowercase words joined by "-"; clips are named `<scene_name>-clip-NN`. */
|
||||
scene_name: string;
|
||||
references: SceneDraftReference[];
|
||||
instructions: string;
|
||||
/** Let the writer download reference photos from Wikimedia Commons for subjects with none. */
|
||||
find_references: boolean;
|
||||
}
|
||||
|
||||
/** A photo the writer downloaded, with where it came from so it can be checked and credited. */
|
||||
export interface FetchedReference {
|
||||
path: string;
|
||||
subject: string;
|
||||
query: string;
|
||||
title: string;
|
||||
/** The Wikimedia Commons file page. */
|
||||
page: string;
|
||||
license: string;
|
||||
artist: string;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/** A drafted job, in the job-file shape: `prompt` is a list of lines. */
|
||||
export type DraftedJob = Omit<CreateJobRequest, "prompt"> & { prompt: string[] };
|
||||
|
||||
export interface SceneDraft {
|
||||
run_id: string;
|
||||
scene_name: string;
|
||||
status: "running" | "completed" | "failed";
|
||||
started_at: number;
|
||||
finished_at: number | null;
|
||||
/** Clips written so far (while running) or in the end; every one passed validation. */
|
||||
jobs: DraftedJob[];
|
||||
error: string | null;
|
||||
final_response?: string | null;
|
||||
fetched_references: FetchedReference[];
|
||||
}
|
||||
|
||||
/** URL of an image the writer downloaded for a draft. */
|
||||
export function getFetchedReferenceUrl(runId: string, ref: FetchedReference): string {
|
||||
const name = ref.path.split("/").pop() ?? "";
|
||||
return `${getApiBaseUrl()}/scene-writer/drafts/${runId}/references/${encodeURIComponent(name)}`;
|
||||
}
|
||||
|
||||
/** Start drafting a scene's jobs; poll `getSceneDraft` until it finishes. */
|
||||
export async function startSceneDraft(req: SceneDraftRequest): Promise<SceneDraft> {
|
||||
const response = await fetch(`${getApiBaseUrl()}/scene-writer/drafts`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(req),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({ detail: "Failed to start the draft" }));
|
||||
throw new Error(typeof error.detail === "string" ? error.detail : "Failed to start the draft");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function getSceneDraft(runId: string): Promise<SceneDraft> {
|
||||
const response = await fetch(`${getApiBaseUrl()}/scene-writer/drafts/${runId}`);
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to fetch the draft");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/** A clip revision: the agent's rewrite of one clip's job from feedback on its take. */
|
||||
export interface ClipRevision {
|
||||
run_id: string;
|
||||
status: "running" | "completed" | "failed";
|
||||
started_at: number;
|
||||
finished_at: number | null;
|
||||
/** the clip's job as it was sent, in job-file form */
|
||||
original: DraftedJob;
|
||||
/** the revised job; null until it is written */
|
||||
job: DraftedJob | null;
|
||||
/** the agent's explanation: the cause it found and each change it made */
|
||||
final_response: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** Start revising a clip; poll `getClipRevision` until it finishes. */
|
||||
export async function startClipRevision(req: {
|
||||
job: Job;
|
||||
feedback: string;
|
||||
neighbors: Job[];
|
||||
}): Promise<ClipRevision> {
|
||||
const response = await fetch(`${getApiBaseUrl()}/scene-writer/revisions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(req),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({ detail: "Failed to start the revision" }));
|
||||
throw new Error(typeof error.detail === "string" ? error.detail : "Failed to start the revision");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function getClipRevision(runId: string): Promise<ClipRevision> {
|
||||
const response = await fetch(`${getApiBaseUrl()}/scene-writer/revisions/${runId}`);
|
||||
if (!response.ok) throw new Error("Failed to load the revision");
|
||||
return response.json();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,478 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import {
|
||||
attachLastClip,
|
||||
attachLastFrame,
|
||||
attachedLastClips,
|
||||
attachedLastFrames,
|
||||
deferredLastClipSource,
|
||||
deferredLastFrameSource,
|
||||
detachLastClip,
|
||||
detachLastFrame,
|
||||
lastClipSourceJobId,
|
||||
lastFrameSourceJobId,
|
||||
whyCannotAttach,
|
||||
whyCannotAttachClip,
|
||||
} from "@/lib/continuation";
|
||||
import { parseH3Prompt } from "@/lib/h3Prompt";
|
||||
import { readField } from "@/lib/h3PromptFields";
|
||||
|
||||
const FRAME = "/data/uploads/last_frames/last_frame_job-5.png";
|
||||
const REFS = [
|
||||
{ source: "/refs/veteran.png", media_type: "image" },
|
||||
{ source: "/refs/young.png", media_type: "image" },
|
||||
];
|
||||
|
||||
const CLIP = [
|
||||
"subject_definitions:",
|
||||
"<Subject 1> is the veteran broker from <Picture 1>: a confident man.",
|
||||
"<Subject 2> is the young broker from <Picture 2>: eager.",
|
||||
"",
|
||||
"summary:",
|
||||
"[reference generation] The target video shows <Subject 1> and <Subject 2> at a lunch.",
|
||||
"",
|
||||
"retention_analysis:",
|
||||
"<Subject 1> (appears in [Shot 1]): fully_preserved - his suit.",
|
||||
"<Subject 2> (appears in [Shot 1]): fully_preserved - his suit.",
|
||||
"",
|
||||
"detailed_description:",
|
||||
"[Shot 1] Close-up shot, dolly. Continuing the same shot on <Subject 1>, still speaking. (S1) <d>[English] And then, well.</d>",
|
||||
"[Shot 2] At 00:04.000, the shot cuts to a wide shot. Both men at the table.",
|
||||
"",
|
||||
"overall_soundscape:",
|
||||
"Muted restaurant ambience.",
|
||||
"",
|
||||
"non_diegetic_music:",
|
||||
"N/A",
|
||||
].join("\n");
|
||||
|
||||
function attach(prompt = CLIP, refs = REFS, frame = FRAME) {
|
||||
const r = attachLastFrame(prompt, refs, frame);
|
||||
if (!r.ok) throw new Error(r.error);
|
||||
return r;
|
||||
}
|
||||
|
||||
describe("attachLastFrame", () => {
|
||||
it("adds the frame as the next image reference and names it by its picture number", () => {
|
||||
const r = attach();
|
||||
expect(r.label).toBe("<Picture 3>");
|
||||
expect(r.references).toEqual([...REFS, { source: FRAME, media_type: "image" }]);
|
||||
});
|
||||
|
||||
it("defines the frame as the first frame of Shot 1, after the existing definitions", () => {
|
||||
const defs = readField(attach().prompt, "subject_definitions") as string;
|
||||
expect(defs.split("\n")).toEqual([
|
||||
"<Subject 1> is the veteran broker from <Picture 1>: a confident man.",
|
||||
"<Subject 2> is the young broker from <Picture 2>: eager.",
|
||||
"<Picture 3> is the first frame of [Shot 1]: the last frame of the previous clip, showing the scene, positions, and lighting exactly as that clip ended.",
|
||||
]);
|
||||
});
|
||||
|
||||
it("tags the summary as keyframe completion alongside its existing task type, and names the frame", () => {
|
||||
const summary = readField(attach().prompt, "summary") as string;
|
||||
expect(summary.startsWith("[reference generation + keyframe completion] The target video shows")).toBe(true);
|
||||
expect(summary.endsWith("The target video starts from <Picture 3>, the last frame of the previous clip.")).toBe(true);
|
||||
});
|
||||
|
||||
it("adds a first-frame line to the retention analysis", () => {
|
||||
const lines = (readField(attach().prompt, "retention_analysis") as string).split("\n");
|
||||
expect(lines).toHaveLength(3);
|
||||
expect(lines[2]).toBe(
|
||||
"<Picture 3> ([Shot 1] first frame): fully_preserved - the scene, positions, and lighting of the previous clip's last frame are retained.",
|
||||
);
|
||||
});
|
||||
|
||||
it("starts the first shot from the frame and leaves the other shots alone", () => {
|
||||
const shots = (readField(attach().prompt, "detailed_description") as string).split("\n");
|
||||
expect(shots[0]).toBe(
|
||||
"[Shot 1] Close-up shot, dolly. The shot begins from <Picture 3>. Continuing the same shot on <Subject 1>, still speaking. (S1) <d>[English] And then, well.</d>",
|
||||
);
|
||||
expect(shots[1]).toBe("[Shot 2] At 00:04.000, the shot cuts to a wide shot. Both men at the table.");
|
||||
});
|
||||
|
||||
it("leaves every other section untouched", () => {
|
||||
const out = attach().prompt;
|
||||
expect(readField(out, "overall_soundscape")).toBe("Muted restaurant ambience.");
|
||||
expect(readField(out, "non_diegetic_music")).toBe("N/A");
|
||||
});
|
||||
|
||||
it("still reads as a six-section prompt", () => {
|
||||
expect(parseH3Prompt(attach().prompt)).not.toBeNull();
|
||||
});
|
||||
|
||||
it("counts only images when numbering, so a video reference doesn't shift the label", () => {
|
||||
const refs = [REFS[0], { source: "/refs/clip.mp4", media_type: "video" }, REFS[1]];
|
||||
expect(attach(CLIP, refs).label).toBe("<Picture 3>");
|
||||
expect(attach(CLIP, [{ source: "/refs/clip.mp4", media_type: "video" }]).label).toBe("<Picture 1>");
|
||||
});
|
||||
|
||||
it("does not mutate what it was given", () => {
|
||||
const refs = [...REFS];
|
||||
attach(CLIP, refs);
|
||||
expect(refs).toEqual(REFS);
|
||||
});
|
||||
});
|
||||
|
||||
describe("attachLastFrame: summary wording", () => {
|
||||
const summaryOf = (summary: string) =>
|
||||
readField(attach(CLIP.replace("[reference generation] The target video shows <Subject 1> and <Subject 2> at a lunch.", summary)).prompt, "summary") as string;
|
||||
|
||||
it("adds the tag when the summary has none", () => {
|
||||
expect(summaryOf("Two men at a lunch.").startsWith("[keyframe completion] Two men at a lunch.")).toBe(true);
|
||||
});
|
||||
|
||||
it("extends a combined tag", () => {
|
||||
expect(summaryOf("[video editing + reference generation] Edit.").startsWith("[video editing + reference generation + keyframe completion] Edit.")).toBe(true);
|
||||
});
|
||||
|
||||
it("does not repeat a tag that is already there", () => {
|
||||
const s = summaryOf("[keyframe completion + reference generation] Both.");
|
||||
expect(s.startsWith("[keyframe completion + reference generation] Both.")).toBe(true);
|
||||
expect(s.match(/keyframe completion/g)).toHaveLength(1);
|
||||
});
|
||||
});
|
||||
|
||||
describe("attachLastFrame: other prompt shapes", () => {
|
||||
it("replaces an N/A subject_definitions instead of appending under it", () => {
|
||||
const out = attach(CLIP.replace(/subject_definitions:\n[\s\S]*?\n\nsummary:/, "subject_definitions:\nN/A\n\nsummary:")).prompt;
|
||||
expect(readField(out, "subject_definitions")).toBe(
|
||||
"<Picture 3> is the first frame of [Shot 1]: the last frame of the previous clip, showing the scene, positions, and lighting exactly as that clip ended.",
|
||||
);
|
||||
});
|
||||
|
||||
it("keeps hand-written shots verbatim, only prefixing the first", () => {
|
||||
const hand = CLIP.replace(
|
||||
/\[Shot 1\][^\n]*\n\[Shot 2\][^\n]*/,
|
||||
"[Shot 1] A handheld medium shot holds <Subject 1> at frame left.\n[Shot 2] At 00:03.800, the shot cuts to a low tracking shot behind him.",
|
||||
);
|
||||
const shots = (readField(attach(hand).prompt, "detailed_description") as string).split("\n");
|
||||
expect(shots).toEqual([
|
||||
"[Shot 1] The shot begins from <Picture 3>. A handheld medium shot holds <Subject 1> at frame left.",
|
||||
"[Shot 2] At 00:03.800, the shot cuts to a low tracking shot behind him.",
|
||||
]);
|
||||
});
|
||||
|
||||
it("adds the sentence to a first shot that is only dialogue", () => {
|
||||
const dialogueOnly = CLIP.replace(
|
||||
"[Shot 1] Close-up shot, dolly. Continuing the same shot on <Subject 1>, still speaking. (S1) <d>[English] And then, well.</d>",
|
||||
"[Shot 1] Wide shot. (S1) <d>[English] Hi.</d>",
|
||||
);
|
||||
const first = (readField(attach(dialogueOnly).prompt, "detailed_description") as string).split("\n")[0];
|
||||
expect(first).toBe("[Shot 1] Wide shot. The shot begins from <Picture 3>. (S1) <d>[English] Hi.</d>");
|
||||
});
|
||||
});
|
||||
|
||||
describe("attachLastFrame: refusals", () => {
|
||||
it("won't attach the same frame twice", () => {
|
||||
const r = attachLastFrame(CLIP, [...REFS, { source: FRAME, media_type: "image" }], FRAME);
|
||||
expect(r).toEqual({ ok: false, error: "That frame is already attached to this clip." });
|
||||
});
|
||||
|
||||
it("won't rewrite a prompt that isn't in the six-section format", () => {
|
||||
const r = attachLastFrame("A toy car drives into a plush dog.", REFS, FRAME);
|
||||
expect(r.ok).toBe(false);
|
||||
expect(!r.ok && r.error).toMatch(/six-section/);
|
||||
});
|
||||
|
||||
it("won't rewrite a base-format prompt either", () => {
|
||||
const base = "For the target video, <Picture 1> is used.\n\nintegrated_multimodal_description: [Shot 1] A.\n\noverall_soundscape: Quiet.";
|
||||
expect(whyCannotAttach(base, REFS)).toMatch(/six-section/);
|
||||
});
|
||||
|
||||
it("stops at nine image references", () => {
|
||||
const nine = Array.from({ length: 9 }, (_, i) => ({ source: `/r/${i}.png`, media_type: "image" }));
|
||||
const r = attachLastFrame(CLIP, nine, FRAME);
|
||||
expect(!r.ok && r.error).toMatch(/at most 9 image/);
|
||||
});
|
||||
|
||||
it("reports a prompt with no shots", () => {
|
||||
const empty = CLIP.replace(/detailed_description:\n[\s\S]*?\n\noverall_soundscape:/, "detailed_description:\n\n\noverall_soundscape:");
|
||||
const r = attachLastFrame(empty, REFS, FRAME);
|
||||
expect(r.ok).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe("recognizing an attached last frame", () => {
|
||||
it("reads the job id out of the saved path", () => {
|
||||
expect(lastFrameSourceJobId(FRAME)).toBe("job-5");
|
||||
expect(lastFrameSourceJobId("last_frame_abc-123.png")).toBe("abc-123");
|
||||
expect(lastFrameSourceJobId("/refs/veteran.png")).toBeNull();
|
||||
expect(lastFrameSourceJobId("/x/not_last_frame_job-5.png")).toBeNull();
|
||||
});
|
||||
|
||||
it("finds last-frame references among a job's references", () => {
|
||||
expect(attachedLastFrames([...REFS, { source: FRAME, media_type: "image" }])).toEqual([{ source: FRAME, jobId: "job-5" }]);
|
||||
expect(attachedLastFrames(REFS)).toEqual([]);
|
||||
expect(attachedLastFrames(null)).toEqual([]);
|
||||
});
|
||||
|
||||
it("reads the job id out of a reference the server resolves when the job runs", () => {
|
||||
expect(deferredLastFrameSource("job-5")).toBe("job-last-frame:job-5");
|
||||
expect(lastFrameSourceJobId(deferredLastFrameSource("job-5"))).toBe("job-5");
|
||||
expect(lastFrameSourceJobId("job-last-frame:")).toBeNull();
|
||||
expect(attachedLastFrames([...REFS, { source: deferredLastFrameSource("job-5"), media_type: "image" }])).toEqual([
|
||||
{ source: "job-last-frame:job-5", jobId: "job-5" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("attaches a deferred frame exactly like a saved one", () => {
|
||||
const result = attachLastFrame(CLIP, REFS, deferredLastFrameSource("job-5"));
|
||||
expect(result.ok).toBe(true);
|
||||
if (result.ok) {
|
||||
expect(result.references.at(-1)).toEqual({ source: "job-last-frame:job-5", media_type: "image" });
|
||||
expect(result.label).toBe("<Picture 3>");
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
describe("detachLastFrame", () => {
|
||||
const roundTrip = (prompt: string, frame = FRAME) => {
|
||||
const attached = attach(prompt, REFS, frame);
|
||||
const r = detachLastFrame(attached.prompt, attached.references, frame);
|
||||
if (!r.ok) throw new Error(r.error);
|
||||
return r;
|
||||
};
|
||||
|
||||
it("takes back exactly what attaching added", () => {
|
||||
const r = roundTrip(CLIP);
|
||||
expect(r.references).toEqual(REFS);
|
||||
expect(r.prompt).toBe(CLIP);
|
||||
});
|
||||
|
||||
it("works for a frame the server resolves when the job runs", () => {
|
||||
expect(roundTrip(CLIP, deferredLastFrameSource("job-5")).prompt).toBe(CLIP);
|
||||
});
|
||||
|
||||
it.each([
|
||||
["a summary with no tag", "Two men at a lunch."],
|
||||
["a combined tag", "[video editing + reference generation] Edit."],
|
||||
])("restores %s", (_name, summary) => {
|
||||
const prompt = CLIP.replace("[reference generation] The target video shows <Subject 1> and <Subject 2> at a lunch.", summary);
|
||||
expect(roundTrip(prompt).prompt).toBe(prompt);
|
||||
});
|
||||
|
||||
it("restores hand-written shots and an N/A section", () => {
|
||||
const hand = CLIP.replace(/\[Shot 1\][^\n]*\n\[Shot 2\][^\n]*/, "[Shot 1] A handheld medium shot.\n[Shot 2] At 00:03.800, the shot cuts to a low tracking shot.").replace(
|
||||
/subject_definitions:\n[\s\S]*?\n\nsummary:/,
|
||||
"subject_definitions:\nN/A\n\nsummary:",
|
||||
);
|
||||
expect(roundTrip(hand).prompt).toBe(hand);
|
||||
});
|
||||
|
||||
it("removes the sentence from a first shot that is only dialogue", () => {
|
||||
const dialogueOnly = CLIP.replace(
|
||||
"[Shot 1] Close-up shot, dolly. Continuing the same shot on <Subject 1>, still speaking. (S1) <d>[English] And then, well.</d>",
|
||||
"[Shot 1] Wide shot. (S1) <d>[English] Hi.</d>",
|
||||
);
|
||||
expect(roundTrip(dialogueOnly).prompt).toBe(dialogueOnly);
|
||||
});
|
||||
|
||||
it("only removes the reference from a prompt that was never rewritten", () => {
|
||||
const r = detachLastFrame("A plain prompt.", [...REFS, { source: FRAME, media_type: "image" }], FRAME);
|
||||
expect(r).toEqual({ ok: true, prompt: "A plain prompt.", references: REFS });
|
||||
});
|
||||
|
||||
it("refuses when other images were added after the frame", () => {
|
||||
const attached = attach();
|
||||
const r = detachLastFrame(attached.prompt, [...attached.references, { source: "/refs/extra.png", media_type: "image" }], FRAME);
|
||||
expect(r.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses a frame that isn't attached", () => {
|
||||
expect(detachLastFrame(CLIP, REFS, FRAME)).toEqual({ ok: false, error: "That frame isn't attached to this clip." });
|
||||
});
|
||||
});
|
||||
|
||||
const CLIP_VIDEO = "/data/uploads/last_clips/last_clip_job-5.mp4";
|
||||
|
||||
function attachClip(prompt = CLIP, refs = REFS, clip = CLIP_VIDEO) {
|
||||
const r = attachLastClip(prompt, refs, clip);
|
||||
if (!r.ok) throw new Error(r.error);
|
||||
return r;
|
||||
}
|
||||
|
||||
describe("attachLastClip", () => {
|
||||
it("adds the clip as the next video reference and names it by its video number", () => {
|
||||
const r = attachClip();
|
||||
expect(r.label).toBe("<Video 1>");
|
||||
expect(r.references).toEqual([...REFS, { source: CLIP_VIDEO, media_type: "video" }]);
|
||||
});
|
||||
|
||||
it("numbers a second video reference after the first", () => {
|
||||
const withOne = attachClip();
|
||||
const r = attachLastClip(withOne.prompt, withOne.references, "/refs/another.mp4");
|
||||
if (!r.ok) throw new Error(r.error);
|
||||
expect(r.label).toBe("<Video 2>");
|
||||
});
|
||||
|
||||
it("declares the video standalone, not bound to a <Subject> the way pictures and audio are", () => {
|
||||
const defs = readField(attachClip().prompt, "subject_definitions") as string;
|
||||
expect(defs).toContain("<Video 1> is the source video for [Shot 1]");
|
||||
expect(defs).not.toMatch(/<Subject \d+> is .*<Video 1>/);
|
||||
});
|
||||
|
||||
it("adds a retention line with the fully_preserved status", () => {
|
||||
const line = (readField(attachClip().prompt, "retention_analysis") as string).split("\n").at(-1);
|
||||
expect(line).toBe(
|
||||
"<Video 1> (continuation source): fully_preserved - the motion, scene, positions, and lighting of the previous clip's ending are continued exactly.",
|
||||
);
|
||||
});
|
||||
|
||||
it("prefixes the first shot with a continuation lead, keeping hand-written shots verbatim otherwise", () => {
|
||||
const hand = CLIP.replace(
|
||||
/\[Shot 1\][^\n]*\n\[Shot 2\][^\n]*/,
|
||||
"[Shot 1] A handheld medium shot holds <Subject 1> at frame left.\n[Shot 2] At 00:03.800, the shot cuts to a low tracking shot.",
|
||||
);
|
||||
const shots = (readField(attachClip(hand).prompt, "detailed_description") as string).split("\n");
|
||||
expect(shots).toEqual([
|
||||
"[Shot 1] The shot continues from <Video 1>. A handheld medium shot holds <Subject 1> at frame left.",
|
||||
"[Shot 2] At 00:03.800, the shot cuts to a low tracking shot.",
|
||||
]);
|
||||
});
|
||||
|
||||
describe("summary wording", () => {
|
||||
const summaryOf = (summary: string) =>
|
||||
readField(attachClip(CLIP.replace("[reference generation] The target video shows <Subject 1> and <Subject 2> at a lunch.", summary)).prompt, "summary") as string;
|
||||
|
||||
it("adds the tag when the summary has none", () => {
|
||||
expect(summaryOf("Two men at a lunch.")).toBe("[video continuation] Two men at a lunch.");
|
||||
});
|
||||
|
||||
it("replaces reference generation, not adds to it", () => {
|
||||
expect(summaryOf("[reference generation] Two men.")).toBe("[video continuation] Two men.");
|
||||
});
|
||||
|
||||
it("keeps a combined audio reference tag, per the guide's documented combination", () => {
|
||||
expect(summaryOf("[reference generation + audio reference] Two men.")).toBe("[video continuation + audio reference] Two men.");
|
||||
});
|
||||
|
||||
it("drops keyframe completion when switching from a picture-based continuation", () => {
|
||||
expect(summaryOf("[reference generation + keyframe completion] Two men.")).toBe("[video continuation] Two men.");
|
||||
});
|
||||
});
|
||||
|
||||
describe("refusals", () => {
|
||||
it("won't attach the same clip twice", () => {
|
||||
const r = attachLastClip(CLIP, [...REFS, { source: CLIP_VIDEO, media_type: "video" }], CLIP_VIDEO);
|
||||
expect(r).toEqual({ ok: false, error: "That clip is already attached to this job." });
|
||||
});
|
||||
|
||||
it("refuses a prompt that is not in the six-section format", () => {
|
||||
const r = attachLastClip("A plain prompt.", REFS, CLIP_VIDEO);
|
||||
expect(r).toEqual({ ok: false, error: "This prompt isn't in the six-section format, so it can't be rewritten automatically. Attach the clip in Edit instead." });
|
||||
});
|
||||
|
||||
it("refuses past H3's video reference limit", () => {
|
||||
const many = Array.from({ length: 3 }, (_, i) => ({ source: `/v${i}.mp4`, media_type: "video" }));
|
||||
const r = attachLastClip(CLIP, [...REFS, ...many], CLIP_VIDEO);
|
||||
expect(r).toEqual({ ok: false, error: "A job can have at most 3 video references." });
|
||||
});
|
||||
|
||||
it("refuses a prompt with no shots", () => {
|
||||
const noShots = CLIP.replace(/detailed_description:\n[\s\S]*?\n\noverall_soundscape:/, "detailed_description:\n\noverall_soundscape:");
|
||||
const r = attachLastClip(noShots, REFS, CLIP_VIDEO);
|
||||
expect(r).toEqual({ ok: false, error: "This prompt has no shots to continue from the clip." });
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe("recognizing an attached last clip", () => {
|
||||
it("reads the job id out of the saved path", () => {
|
||||
expect(lastClipSourceJobId(CLIP_VIDEO)).toBe("job-5");
|
||||
expect(lastClipSourceJobId("last_clip_abc-123.mp4")).toBe("abc-123");
|
||||
expect(lastClipSourceJobId("/refs/veteran.png")).toBeNull();
|
||||
});
|
||||
|
||||
it("reads the job id out of a reference the server resolves when the job runs", () => {
|
||||
expect(deferredLastClipSource("job-5")).toBe("job-last-clip:job-5");
|
||||
expect(lastClipSourceJobId(deferredLastClipSource("job-5"))).toBe("job-5");
|
||||
});
|
||||
|
||||
it("finds last-clip references among a job's references, separately from last-frame ones", () => {
|
||||
const frame = { source: "/data/uploads/last_frames/last_frame_job-2.png", media_type: "image" };
|
||||
expect(attachedLastClips([...REFS, frame, { source: CLIP_VIDEO, media_type: "video" }])).toEqual([
|
||||
{ source: CLIP_VIDEO, jobId: "job-5" },
|
||||
]);
|
||||
expect(attachedLastClips([frame])).toEqual([]);
|
||||
expect(attachedLastClips(null)).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
describe("whyCannotAttachClip", () => {
|
||||
it("blocks a prompt not in the six-section format", () => {
|
||||
expect(whyCannotAttachClip("A plain prompt.", REFS)).toMatch(/six-section/);
|
||||
});
|
||||
|
||||
it("allows a normal prompt under the video limit", () => {
|
||||
expect(whyCannotAttachClip(CLIP, REFS)).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("detachLastClip", () => {
|
||||
const roundTrip = (prompt: string, clip = CLIP_VIDEO) => {
|
||||
const attached = attachClip(prompt, REFS, clip);
|
||||
const r = detachLastClip(attached.prompt, attached.references, clip);
|
||||
if (!r.ok) throw new Error(r.error);
|
||||
return r;
|
||||
};
|
||||
|
||||
it("takes back exactly what attaching added", () => {
|
||||
const r = roundTrip(CLIP);
|
||||
expect(r.references).toEqual(REFS);
|
||||
expect(r.prompt).toBe(CLIP);
|
||||
});
|
||||
|
||||
it("works for a clip the server resolves when the job runs", () => {
|
||||
expect(roundTrip(CLIP, deferredLastClipSource("job-5")).prompt).toBe(CLIP);
|
||||
});
|
||||
|
||||
it("round-trips a starting tag combined with audio reference", () => {
|
||||
// A realistic starting point: audio references already attached (see continuation.test.ts's
|
||||
// own audio-reference wiring), before any video continuation is attached.
|
||||
const prompt = CLIP.replace(
|
||||
"[reference generation] The target video shows <Subject 1> and <Subject 2> at a lunch.",
|
||||
"[reference generation + audio reference] Two men at a lunch.",
|
||||
);
|
||||
expect(roundTrip(prompt).prompt).toBe(prompt);
|
||||
});
|
||||
|
||||
// Unlike attachLastFrame's "keyframe completion" (only ever added alongside the existing base
|
||||
// tag), "video continuation" *replaces* the base tag -- so once attached, nothing remembers
|
||||
// what was there before, and detaching lands on the default "reference generation", not back
|
||||
// on "no tag".
|
||||
it("restores a summary with no tag to the default reference-generation tag, not back to no tag", () => {
|
||||
const prompt = CLIP.replace("[reference generation] The target video shows <Subject 1> and <Subject 2> at a lunch.", "Two men at a lunch.");
|
||||
expect(roundTrip(prompt).prompt).toBe(CLIP.replace("The target video shows <Subject 1> and <Subject 2> at a lunch.", "Two men at a lunch."));
|
||||
});
|
||||
|
||||
it("restores hand-written shots and an N/A section", () => {
|
||||
const hand = CLIP.replace(/\[Shot 1\][^\n]*\n\[Shot 2\][^\n]*/, "[Shot 1] A handheld medium shot.\n[Shot 2] At 00:03.800, the shot cuts to a low tracking shot.").replace(
|
||||
/subject_definitions:\n[\s\S]*?\n\nsummary:/,
|
||||
"subject_definitions:\nN/A\n\nsummary:",
|
||||
);
|
||||
expect(roundTrip(hand).prompt).toBe(hand);
|
||||
});
|
||||
|
||||
it("only removes the reference from a prompt that was never rewritten", () => {
|
||||
const r = detachLastClip("A plain prompt.", [...REFS, { source: CLIP_VIDEO, media_type: "video" }], CLIP_VIDEO);
|
||||
expect(r).toEqual({ ok: true, prompt: "A plain prompt.", references: REFS });
|
||||
});
|
||||
|
||||
it("refuses when other videos were added after the clip", () => {
|
||||
const attached = attachClip();
|
||||
const r = detachLastClip(attached.prompt, [...attached.references, { source: "/refs/extra.mp4", media_type: "video" }], CLIP_VIDEO);
|
||||
expect(r.ok).toBe(false);
|
||||
});
|
||||
|
||||
it("refuses a clip that isn't attached", () => {
|
||||
expect(detachLastClip(CLIP, REFS, CLIP_VIDEO)).toEqual({ ok: false, error: "That clip isn't attached to this job." });
|
||||
});
|
||||
|
||||
it("does not remove a last-frame attachment, and vice versa", () => {
|
||||
const withFrame = attach(CLIP, REFS, FRAME);
|
||||
const withBoth = attachClip(withFrame.prompt, withFrame.references, CLIP_VIDEO);
|
||||
const r = detachLastClip(withBoth.prompt, withBoth.references, CLIP_VIDEO);
|
||||
if (!r.ok) throw new Error(r.error);
|
||||
expect(attachedLastFrames(r.references)).toEqual([{ source: FRAME, jobId: "job-5" }]);
|
||||
expect(attachedLastClips(r.references)).toEqual([]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,334 @@
|
||||
/**
|
||||
* Start a clip from how the one before it ended -- either its last frame (a
|
||||
* still) or its last second or so (a short video).
|
||||
*
|
||||
* A scene is generated as many short clips. To make one pick up where the
|
||||
* previous ended, the previous clip's ending is attached as a reference, and
|
||||
* the prompt is rewritten the way the prompt guide describes for a picture
|
||||
* that anchors the opening frame (`attachLastFrame`) or a video that the shot
|
||||
* continues from (`attachLastClip`). The wording is kept together here so it
|
||||
* is easy to change if the guide is read differently.
|
||||
*
|
||||
* The two approaches trade off differently: a still is a hard, exact opening
|
||||
* frame, but the model has only a pose to guess the continuing motion from. A
|
||||
* short video carries real motion, at the cost of one more reference slot and
|
||||
* (per H3's own limits) at most 3 per job instead of 9.
|
||||
*/
|
||||
import { readField, writeField } from "@/lib/h3PromptFields";
|
||||
import { H3_REFERENCE_LIMITS } from "@/lib/h3References";
|
||||
import {
|
||||
detailedDescriptionFromShots,
|
||||
newTextPart,
|
||||
parseDetailedDescription,
|
||||
} from "@/lib/h3Shots";
|
||||
|
||||
export interface Reference {
|
||||
source: string;
|
||||
media_type: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A reference that means "the last frame of that job's video", resolved by the
|
||||
* server when this job runs -- so it can be attached before that job has
|
||||
* finished, and the server's queue knows this job must wait for it.
|
||||
*/
|
||||
export const DEFERRED_LAST_FRAME_PREFIX = "job-last-frame:";
|
||||
/** As above, but for the previous job's trailing video instead of its last frame. */
|
||||
export const DEFERRED_LAST_CLIP_PREFIX = "job-last-clip:";
|
||||
|
||||
export const deferredLastFrameSource = (jobId: string) => `${DEFERRED_LAST_FRAME_PREFIX}${jobId}`;
|
||||
export const deferredLastClipSource = (jobId: string) => `${DEFERRED_LAST_CLIP_PREFIX}${jobId}`;
|
||||
|
||||
/** Saved by the server as `.../last_frames/last_frame_<job id>.png`. */
|
||||
const LAST_FRAME_RE = /(?:^|\/)last_frame_([^/]+)\.png$/;
|
||||
/** Saved by the server as `.../last_clips/last_clip_<job id>.mp4`. */
|
||||
const LAST_CLIP_RE = /(?:^|\/)last_clip_([^/]+)\.mp4$/;
|
||||
|
||||
const REQUIRED_SECTIONS = [
|
||||
"subject_definitions",
|
||||
"summary",
|
||||
"retention_analysis",
|
||||
"detailed_description",
|
||||
] as const;
|
||||
|
||||
/** The job a last-frame reference came from, or null if `source` isn't one. */
|
||||
export function lastFrameSourceJobId(source: string): string | null {
|
||||
if (source.startsWith(DEFERRED_LAST_FRAME_PREFIX)) return source.slice(DEFERRED_LAST_FRAME_PREFIX.length) || null;
|
||||
return source.match(LAST_FRAME_RE)?.[1] ?? null;
|
||||
}
|
||||
|
||||
/** Last-frame references already on a job, with the job each came from. */
|
||||
export function attachedLastFrames(references: Reference[] | null | undefined): { source: string; jobId: string }[] {
|
||||
return (references ?? []).flatMap((r) => {
|
||||
const jobId = lastFrameSourceJobId(r.source);
|
||||
return jobId ? [{ source: r.source, jobId }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
/** The job a last-clip reference came from, or null if `source` isn't one. */
|
||||
export function lastClipSourceJobId(source: string): string | null {
|
||||
if (source.startsWith(DEFERRED_LAST_CLIP_PREFIX)) return source.slice(DEFERRED_LAST_CLIP_PREFIX.length) || null;
|
||||
return source.match(LAST_CLIP_RE)?.[1] ?? null;
|
||||
}
|
||||
|
||||
/** Last-clip references already on a job, with the job each came from. */
|
||||
export function attachedLastClips(references: Reference[] | null | undefined): { source: string; jobId: string }[] {
|
||||
return (references ?? []).flatMap((r) => {
|
||||
const jobId = lastClipSourceJobId(r.source);
|
||||
return jobId ? [{ source: r.source, jobId }] : [];
|
||||
});
|
||||
}
|
||||
|
||||
/** Why a last frame can't be attached to this job, or null if it can. */
|
||||
export function whyCannotAttach(prompt: string, references: Reference[] | null | undefined): string | null {
|
||||
const missing = REQUIRED_SECTIONS.filter((s) => readField(prompt, s) === null);
|
||||
if (missing.length > 0) {
|
||||
return "This prompt isn't in the six-section format, so it can't be rewritten automatically. Attach the frame in Edit instead.";
|
||||
}
|
||||
const images = (references ?? []).filter((r) => r.media_type === "image").length;
|
||||
if (images >= H3_REFERENCE_LIMITS.image) {
|
||||
return `A job can have at most ${H3_REFERENCE_LIMITS.image} image references.`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** "[reference generation]" -> "[reference generation + keyframe completion]"; no tag -> "[keyframe completion] ...". */
|
||||
function withKeyframeCompletion(summary: string): string {
|
||||
const tag = summary.match(/^\[([^\]]*)\]\s*/);
|
||||
if (!tag) return `[keyframe completion] ${summary}`.trim();
|
||||
const types = tag[1].split("+").map((t) => t.trim()).filter(Boolean);
|
||||
if (types.includes("keyframe completion")) return summary;
|
||||
return `[${[...types, "keyframe completion"].join(" + ")}] ${summary.slice(tag[0].length)}`.trim();
|
||||
}
|
||||
|
||||
const isBlank = (text: string) => !text.trim() || text.trim().toUpperCase() === "N/A";
|
||||
const appendLine = (body: string, line: string) => (isBlank(body) ? line : `${body}\n${line}`);
|
||||
|
||||
export type AttachResult =
|
||||
| { ok: true; prompt: string; references: Reference[]; label: string }
|
||||
| { ok: false; error: string };
|
||||
|
||||
/**
|
||||
* Add `framePath` as the next image reference and rewrite the prompt to use it
|
||||
* as the opening frame of Shot 1. Fails, changing nothing, if the frame is
|
||||
* already attached or the prompt isn't in a form that can be rewritten safely.
|
||||
*/
|
||||
export function attachLastFrame(prompt: string, references: Reference[], framePath: string): AttachResult {
|
||||
if (references.some((r) => r.source === framePath)) {
|
||||
return { ok: false, error: "That frame is already attached to this clip." };
|
||||
}
|
||||
const blocked = whyCannotAttach(prompt, references);
|
||||
if (blocked) return { ok: false, error: blocked };
|
||||
|
||||
// Labels count per media type in reference order, so the new image is the next <Picture N>.
|
||||
const label = `<Picture ${references.filter((r) => r.media_type === "image").length + 1}>`;
|
||||
|
||||
const { preamble, shots } = parseDetailedDescription(readField(prompt, "detailed_description") ?? "", []);
|
||||
if (shots.length === 0) return { ok: false, error: "This prompt has no shots to start from the frame." };
|
||||
const first = shots[0];
|
||||
const firstText = first.body.find((p) => p.kind === "text");
|
||||
if (firstText && firstText.kind === "text") firstText.text = `The shot begins from ${label}. ${firstText.text}`;
|
||||
else first.body.unshift(newTextPart(`The shot begins from ${label}.`));
|
||||
|
||||
let next = prompt;
|
||||
next = writeField(
|
||||
next,
|
||||
"subject_definitions",
|
||||
appendLine(
|
||||
readField(next, "subject_definitions") ?? "",
|
||||
`${label} is the first frame of [Shot 1]: the last frame of the previous clip, showing the scene, positions, and lighting exactly as that clip ended.`,
|
||||
),
|
||||
);
|
||||
next = writeField(
|
||||
next,
|
||||
"summary",
|
||||
`${withKeyframeCompletion(readField(next, "summary") ?? "")} The target video starts from ${label}, the last frame of the previous clip.`,
|
||||
);
|
||||
next = writeField(
|
||||
next,
|
||||
"retention_analysis",
|
||||
appendLine(
|
||||
readField(next, "retention_analysis") ?? "",
|
||||
`${label} ([Shot 1] first frame): fully_preserved - the scene, positions, and lighting of the previous clip's last frame are retained.`,
|
||||
),
|
||||
);
|
||||
next = writeField(next, "detailed_description", detailedDescriptionFromShots(shots, preamble));
|
||||
|
||||
return { ok: true, prompt: next, references: [...references, { source: framePath, media_type: "image" }], label };
|
||||
}
|
||||
|
||||
/** Drop the lines `attachLastFrame` added, leaving "N/A" if nothing else was there. */
|
||||
function removeLine(body: string, startsWith: string): string {
|
||||
const kept = body.split("\n").filter((l) => !l.trim().startsWith(startsWith));
|
||||
const text = kept.join("\n").trim();
|
||||
return text || "N/A";
|
||||
}
|
||||
|
||||
export type DetachResult = { ok: true; prompt: string; references: Reference[] } | { ok: false; error: string };
|
||||
|
||||
/**
|
||||
* Undo `attachLastFrame`: remove the reference at `source` and take back what was
|
||||
* added to the prompt for it. Only the most recently added image can be removed --
|
||||
* removing an earlier one would renumber the pictures after it.
|
||||
*/
|
||||
export function detachLastFrame(prompt: string, references: Reference[], source: string): DetachResult {
|
||||
const images = references.filter((r) => r.media_type === "image");
|
||||
const index = images.findIndex((r) => r.source === source);
|
||||
if (index < 0) return { ok: false, error: "That frame isn't attached to this clip." };
|
||||
if (index !== images.length - 1) {
|
||||
return { ok: false, error: "Other images were added after that frame. Remove it in Edit instead." };
|
||||
}
|
||||
const remaining = references.filter((r) => r.source !== source);
|
||||
|
||||
// A prompt that isn't in the six-section form was never rewritten; only the reference goes.
|
||||
if (REQUIRED_SECTIONS.some((s) => readField(prompt, s) === null)) {
|
||||
return { ok: true, prompt, references: remaining };
|
||||
}
|
||||
|
||||
const label = `<Picture ${index + 1}>`;
|
||||
let next = prompt;
|
||||
next = writeField(next, "subject_definitions", removeLine(readField(next, "subject_definitions") ?? "", `${label} is the first frame of [Shot 1]`));
|
||||
next = writeField(next, "retention_analysis", removeLine(readField(next, "retention_analysis") ?? "", `${label} ([Shot 1] first frame)`));
|
||||
|
||||
const summary = readField(next, "summary") ?? "";
|
||||
const sentence = ` The target video starts from ${label}, the last frame of the previous clip.`;
|
||||
if (summary.includes(sentence)) {
|
||||
let restored = summary.replace(sentence, "");
|
||||
const tag = restored.match(/^\[([^\]]*)\]\s*/);
|
||||
if (tag) {
|
||||
const types = tag[1].split("+").map((t) => t.trim()).filter((t) => t && t !== "keyframe completion");
|
||||
restored = types.length > 0 ? `[${types.join(" + ")}] ${restored.slice(tag[0].length)}` : restored.slice(tag[0].length);
|
||||
}
|
||||
next = writeField(next, "summary", restored.trim());
|
||||
}
|
||||
|
||||
const { preamble, shots } = parseDetailedDescription(readField(next, "detailed_description") ?? "", []);
|
||||
const lead = `The shot begins from ${label}.`;
|
||||
const first = shots[0];
|
||||
const part = first?.body.find((p) => p.kind === "text" && p.text.startsWith(lead));
|
||||
if (first && part && part.kind === "text") {
|
||||
part.text = part.text.slice(lead.length).trimStart();
|
||||
if (!part.text) first.body = first.body.filter((p) => p !== part);
|
||||
next = writeField(next, "detailed_description", detailedDescriptionFromShots(shots, preamble));
|
||||
}
|
||||
return { ok: true, prompt: next, references: remaining };
|
||||
}
|
||||
|
||||
/** Why a trailing clip can't be attached to this job, or null if it can. */
|
||||
export function whyCannotAttachClip(prompt: string, references: Reference[] | null | undefined): string | null {
|
||||
const missing = REQUIRED_SECTIONS.filter((s) => readField(prompt, s) === null);
|
||||
if (missing.length > 0) {
|
||||
return "This prompt isn't in the six-section format, so it can't be rewritten automatically. Attach the clip in Edit instead.";
|
||||
}
|
||||
const videos = (references ?? []).filter((r) => r.media_type === "video").length;
|
||||
if (videos >= H3_REFERENCE_LIMITS.video) {
|
||||
return `A job can have at most ${H3_REFERENCE_LIMITS.video} video references.`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** "[reference generation + audio reference]" -> "[video continuation + audio reference]"; drops "keyframe completion". */
|
||||
function withVideoContinuation(summary: string): string {
|
||||
const tag = summary.match(/^\[([^\]]*)\]\s*/);
|
||||
const rest = tag ? summary.slice(tag[0].length) : summary;
|
||||
const kept = tag
|
||||
? tag[1].split("+").map((t) => t.trim()).filter((t) => t && t !== "reference generation" && t !== "keyframe completion")
|
||||
: [];
|
||||
if (!kept.includes("video continuation")) kept.unshift("video continuation");
|
||||
return `[${kept.join(" + ")}] ${rest}`.trim();
|
||||
}
|
||||
|
||||
/**
|
||||
* Add `clipPath` as the next video reference and rewrite the prompt so Shot 1
|
||||
* continues from it, per the guide's video-reference form: a standalone
|
||||
* `<Video N>` declaration (videos aren't bound to a `<Subject>` the way
|
||||
* pictures and audio are), a `[video continuation]` task tag, a retention
|
||||
* line, and "the shot continues from <Video N>" in the first shot.
|
||||
*/
|
||||
export function attachLastClip(prompt: string, references: Reference[], clipPath: string): AttachResult {
|
||||
if (references.some((r) => r.source === clipPath)) {
|
||||
return { ok: false, error: "That clip is already attached to this job." };
|
||||
}
|
||||
const blocked = whyCannotAttachClip(prompt, references);
|
||||
if (blocked) return { ok: false, error: blocked };
|
||||
|
||||
const label = `<Video ${references.filter((r) => r.media_type === "video").length + 1}>`;
|
||||
|
||||
const { preamble, shots } = parseDetailedDescription(readField(prompt, "detailed_description") ?? "", []);
|
||||
if (shots.length === 0) return { ok: false, error: "This prompt has no shots to continue from the clip." };
|
||||
const first = shots[0];
|
||||
const lead = `The shot continues from ${label}.`;
|
||||
const firstText = first.body.find((p) => p.kind === "text");
|
||||
if (firstText && firstText.kind === "text") firstText.text = `${lead} ${firstText.text}`;
|
||||
else first.body.unshift(newTextPart(lead));
|
||||
|
||||
let next = prompt;
|
||||
next = writeField(
|
||||
next,
|
||||
"subject_definitions",
|
||||
appendLine(
|
||||
readField(next, "subject_definitions") ?? "",
|
||||
`${label} is the source video for [Shot 1]: the last second of the previous clip, providing the continuation starting point and its motion.`,
|
||||
),
|
||||
);
|
||||
next = writeField(next, "summary", withVideoContinuation(readField(next, "summary") ?? ""));
|
||||
next = writeField(
|
||||
next,
|
||||
"retention_analysis",
|
||||
appendLine(
|
||||
readField(next, "retention_analysis") ?? "",
|
||||
`${label} (continuation source): fully_preserved - the motion, scene, positions, and lighting of the previous clip's ending are continued exactly.`,
|
||||
),
|
||||
);
|
||||
next = writeField(next, "detailed_description", detailedDescriptionFromShots(shots, preamble));
|
||||
|
||||
return { ok: true, prompt: next, references: [...references, { source: clipPath, media_type: "video" }], label };
|
||||
}
|
||||
|
||||
/**
|
||||
* Undo `attachLastClip`: remove the reference at `source` and take back what
|
||||
* was added to the prompt for it. Only the most recently added video can be
|
||||
* removed -- removing an earlier one would renumber the videos after it.
|
||||
*/
|
||||
export function detachLastClip(prompt: string, references: Reference[], source: string): DetachResult {
|
||||
const videos = references.filter((r) => r.media_type === "video");
|
||||
const index = videos.findIndex((r) => r.source === source);
|
||||
if (index < 0) return { ok: false, error: "That clip isn't attached to this job." };
|
||||
if (index !== videos.length - 1) {
|
||||
return { ok: false, error: "Other videos were added after that clip. Remove it in Edit instead." };
|
||||
}
|
||||
const remaining = references.filter((r) => r.source !== source);
|
||||
|
||||
if (REQUIRED_SECTIONS.some((s) => readField(prompt, s) === null)) {
|
||||
return { ok: true, prompt, references: remaining };
|
||||
}
|
||||
|
||||
const label = `<Video ${index + 1}>`;
|
||||
let next = prompt;
|
||||
next = writeField(next, "subject_definitions", removeLine(readField(next, "subject_definitions") ?? "", `${label} is the source video for [Shot 1]`));
|
||||
next = writeField(next, "retention_analysis", removeLine(readField(next, "retention_analysis") ?? "", `${label} (continuation source)`));
|
||||
|
||||
// "video continuation" replaced "reference generation" as the base tag (attachLastClip drops
|
||||
// it outright, unlike "keyframe completion" which is only ever added alongside it) -- so
|
||||
// undoing it means putting "reference generation" back, not just deleting the tag.
|
||||
const summaryTag = (readField(next, "summary") ?? "").match(/^\[([^\]]*)\]\s*/);
|
||||
if (summaryTag && summaryTag[1].split("+").map((t) => t.trim()).includes("video continuation")) {
|
||||
const rest = (readField(next, "summary") ?? "").slice(summaryTag[0].length);
|
||||
const types = summaryTag[1]
|
||||
.split("+")
|
||||
.map((t) => t.trim())
|
||||
.map((t) => (t === "video continuation" ? "reference generation" : t));
|
||||
next = writeField(next, "summary", `[${types.join(" + ")}] ${rest}`.trim());
|
||||
}
|
||||
|
||||
const { preamble, shots } = parseDetailedDescription(readField(next, "detailed_description") ?? "", []);
|
||||
const lead = `The shot continues from ${label}.`;
|
||||
const first = shots[0];
|
||||
const part = first?.body.find((p) => p.kind === "text" && p.text.startsWith(lead));
|
||||
if (first && part && part.kind === "text") {
|
||||
part.text = part.text.slice(lead.length).trimStart();
|
||||
if (!part.text) first.body = first.body.filter((p) => p !== part);
|
||||
next = writeField(next, "detailed_description", detailedDescriptionFromShots(shots, preamble));
|
||||
}
|
||||
return { ok: true, prompt: next, references: remaining };
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { applyGradeInPlace, isNeutralGrade, NEUTRAL_GRADE } from './grading';
|
||||
|
||||
function solidRgba(rgb: [number, number, number], count = 4, alpha = 255): Uint8ClampedArray {
|
||||
const data = new Uint8ClampedArray(count * 4);
|
||||
for (let i = 0; i < count; i++) {
|
||||
data[i * 4] = rgb[0];
|
||||
data[i * 4 + 1] = rgb[1];
|
||||
data[i * 4 + 2] = rgb[2];
|
||||
data[i * 4 + 3] = alpha;
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
function meanRgb(data: Uint8ClampedArray): [number, number, number] {
|
||||
let r = 0, g = 0, b = 0;
|
||||
const n = data.length / 4;
|
||||
for (let i = 0; i < data.length; i += 4) {
|
||||
r += data[i];
|
||||
g += data[i + 1];
|
||||
b += data[i + 2];
|
||||
}
|
||||
return [r / n, g / n, b / n];
|
||||
}
|
||||
|
||||
describe('isNeutralGrade', () => {
|
||||
it('is true only for brightness 0, contrast 1, saturation 1', () => {
|
||||
expect(isNeutralGrade(0, 1, 1)).toBe(true);
|
||||
expect(isNeutralGrade(NEUTRAL_GRADE.brightness, NEUTRAL_GRADE.contrast, NEUTRAL_GRADE.saturation)).toBe(true);
|
||||
expect(isNeutralGrade(1, 1, 1)).toBe(false);
|
||||
expect(isNeutralGrade(0, 1.1, 1)).toBe(false);
|
||||
expect(isNeutralGrade(0, 1, 0.9)).toBe(false);
|
||||
});
|
||||
});
|
||||
|
||||
describe('applyGradeInPlace', () => {
|
||||
it('leaves neutral values as a no-op', () => {
|
||||
const data = solidRgba([200, 100, 50]);
|
||||
const before = Array.from(data);
|
||||
applyGradeInPlace(data, 0, 1, 1);
|
||||
expect(Array.from(data)).toEqual(before);
|
||||
});
|
||||
|
||||
it('leaves alpha untouched', () => {
|
||||
const data = solidRgba([200, 100, 50], 1, 137);
|
||||
applyGradeInPlace(data, 50, 1.5, 0.5);
|
||||
expect(data[3]).toBe(137);
|
||||
});
|
||||
|
||||
it('positive brightness lightens every channel', () => {
|
||||
const data = solidRgba([200, 100, 50]);
|
||||
const [r0, g0, b0] = meanRgb(data);
|
||||
applyGradeInPlace(data, 50, 1, 1);
|
||||
const [r1, g1, b1] = meanRgb(data);
|
||||
expect(r1).toBeGreaterThan(r0);
|
||||
expect(g1).toBeGreaterThan(g0);
|
||||
expect(b1).toBeGreaterThan(b0);
|
||||
});
|
||||
|
||||
it('negative brightness darkens', () => {
|
||||
const data = solidRgba([200, 100, 50]);
|
||||
const [r0, g0, b0] = meanRgb(data);
|
||||
applyGradeInPlace(data, -50, 1, 1);
|
||||
const [r1, g1, b1] = meanRgb(data);
|
||||
expect(r1).toBeLessThan(r0);
|
||||
expect(g1).toBeLessThan(g0);
|
||||
expect(b1).toBeLessThan(b0);
|
||||
});
|
||||
|
||||
it('clamps brightness at the top of the range instead of wrapping', () => {
|
||||
const data = solidRgba([240, 240, 240]);
|
||||
applyGradeInPlace(data, 100, 1, 1);
|
||||
expect(Array.from(data.subarray(0, 3))).toEqual([255, 255, 255]);
|
||||
});
|
||||
|
||||
it('clamps brightness at the bottom of the range instead of wrapping', () => {
|
||||
const data = solidRgba([10, 10, 10]);
|
||||
applyGradeInPlace(data, -100, 1, 1);
|
||||
expect(Array.from(data.subarray(0, 3))).toEqual([0, 0, 0]);
|
||||
});
|
||||
|
||||
it('zero saturation makes every channel converge on the luma', () => {
|
||||
const data = solidRgba([200, 100, 50]);
|
||||
applyGradeInPlace(data, 0, 1, 0);
|
||||
const [r, g, b] = meanRgb(data);
|
||||
expect(Math.abs(r - g)).toBeLessThan(2);
|
||||
expect(Math.abs(g - b)).toBeLessThan(2);
|
||||
});
|
||||
|
||||
it('saturation above one widens the channel spread', () => {
|
||||
const base = solidRgba([200, 100, 50]);
|
||||
const [baseR, , baseB] = meanRgb(base);
|
||||
const data = solidRgba([200, 100, 50]);
|
||||
applyGradeInPlace(data, 0, 1, 2);
|
||||
const [r, , b] = meanRgb(data);
|
||||
expect(r - b).toBeGreaterThan(baseR - baseB);
|
||||
});
|
||||
|
||||
it('high contrast pushes values away from mid-gray, in both directions', () => {
|
||||
const base = solidRgba([200, 100, 50]);
|
||||
const [baseR, baseG, baseB] = meanRgb(base);
|
||||
const data = solidRgba([200, 100, 50]);
|
||||
applyGradeInPlace(data, 0, 2, 1);
|
||||
const [r, g, b] = meanRgb(data);
|
||||
expect(r).toBeGreaterThan(baseR); // 200 sits above 128, pushed further up
|
||||
expect(g).toBeLessThan(baseG); // 100 sits below 128, pushed further down
|
||||
expect(b).toBeLessThan(baseB); // 50 sits below 128, pushed further down
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
/**
|
||||
* Client-side mirror of trim.py's `_apply_grade`, so the "before/after" preview in
|
||||
* the Edit video dialog can show the effect of a color grade instantly, without a
|
||||
* server round trip. Same bounds, same order of operations (saturation, then
|
||||
* contrast, then brightness). Not pixel-exact with the server's numpy path --
|
||||
* `Uint8ClampedArray` rounds to the nearest integer on assignment where numpy
|
||||
* truncates -- but the difference is well under a visible unit per channel, and
|
||||
* this is only ever a preview; the server's render (via `POST .../trim`) is what
|
||||
* actually gets saved.
|
||||
*/
|
||||
|
||||
export const BRIGHTNESS_BOUNDS = [-100, 100] as const;
|
||||
export const CONTRAST_BOUNDS = [0, 3] as const;
|
||||
export const SATURATION_BOUNDS = [0, 3] as const;
|
||||
|
||||
export const NEUTRAL_GRADE = { brightness: 0, contrast: 1, saturation: 1 } as const;
|
||||
|
||||
// Perceptual luma weights, matching trim.py's _LUMA_WEIGHTS.
|
||||
const LUMA_R = 0.2126;
|
||||
const LUMA_G = 0.7152;
|
||||
const LUMA_B = 0.0722;
|
||||
|
||||
/** True for the values that leave every pixel unchanged -- mirrors trim.py's `graded` flag. */
|
||||
export function isNeutralGrade(brightness: number, contrast: number, saturation: number): boolean {
|
||||
return brightness === NEUTRAL_GRADE.brightness && contrast === NEUTRAL_GRADE.contrast && saturation === NEUTRAL_GRADE.saturation;
|
||||
}
|
||||
|
||||
/**
|
||||
* Adjusts an RGBA buffer (as in `ImageData.data`) in place; alpha is left alone.
|
||||
* Skipping the call entirely when `isNeutralGrade` is true (as callers should)
|
||||
* avoids the per-pixel work for the common no-op case.
|
||||
*/
|
||||
export function applyGradeInPlace(data: Uint8ClampedArray, brightness: number, contrast: number, saturation: number): void {
|
||||
for (let i = 0; i < data.length; i += 4) {
|
||||
let r = data[i];
|
||||
let g = data[i + 1];
|
||||
let b = data[i + 2];
|
||||
|
||||
if (saturation !== 1) {
|
||||
const luma = r * LUMA_R + g * LUMA_G + b * LUMA_B;
|
||||
r = luma + (r - luma) * saturation;
|
||||
g = luma + (g - luma) * saturation;
|
||||
b = luma + (b - luma) * saturation;
|
||||
}
|
||||
if (contrast !== 1) {
|
||||
r = (r - 128) * contrast + 128;
|
||||
g = (g - 128) * contrast + 128;
|
||||
b = (b - 128) * contrast + 128;
|
||||
}
|
||||
if (brightness !== 0) {
|
||||
r += brightness;
|
||||
g += brightness;
|
||||
b += brightness;
|
||||
}
|
||||
|
||||
// Assigning into a Uint8ClampedArray clamps to [0, 255] and rounds on its own.
|
||||
data[i] = r;
|
||||
data[i + 1] = g;
|
||||
data[i + 2] = b;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
/**
|
||||
* Ballpark spoken-duration estimate for MiniMax-H3 dialogue text, plus the
|
||||
* lexer/tree used to render a styled "performance preview" of the tags.
|
||||
*
|
||||
* Numbers here are heuristic (syllable counting + fixed per-tag seconds from
|
||||
* `H3_TAGS[].seconds`), not measured against the model -- good enough to
|
||||
* flag "this clip is clearly too long for one generation," calibrated
|
||||
* against a real render via `calibrationFactor`.
|
||||
*/
|
||||
|
||||
import { H3_TAGS, type H3Tag } from "@/lib/h3Tags";
|
||||
|
||||
/** Bare tag name, e.g. "<long pause>" -> "long pause". */
|
||||
function bareName(bracketed: string): string {
|
||||
return bracketed.replace(/[<>/]/g, "").trim();
|
||||
}
|
||||
|
||||
const WRAP_TAG_NAMES = new Set(H3_TAGS.filter((t) => t.kind === "wrap").map((t) => bareName(t.open)));
|
||||
const TAG_SECONDS: Record<string, number> = Object.fromEntries(
|
||||
H3_TAGS.map((t) => [bareName(t.open), t.seconds ?? 0]),
|
||||
);
|
||||
/** Bare tag name -> its registry entry, so callers (the preview renderer) can look up category/description. */
|
||||
export const TAG_BY_NAME: Record<string, H3Tag> = Object.fromEntries(H3_TAGS.map((t) => [bareName(t.open), t]));
|
||||
|
||||
/* -------------------------------------------------------------------- */
|
||||
/* Lexer / tree — mirrors the tag registry so preview & estimate always */
|
||||
/* agree with what the console/editor can actually insert. */
|
||||
/* -------------------------------------------------------------------- */
|
||||
|
||||
export type LexToken =
|
||||
| { t: "text"; v: string }
|
||||
| { t: "point"; name: string }
|
||||
| { t: "open"; name: string }
|
||||
| { t: "close"; name: string };
|
||||
|
||||
const TAG_RE = /<(\/?)([a-zA-Z ]+?)>/g;
|
||||
|
||||
export function lexDialogue(text: string): LexToken[] {
|
||||
const tokens: LexToken[] = [];
|
||||
let last = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
TAG_RE.lastIndex = 0;
|
||||
while ((m = TAG_RE.exec(text))) {
|
||||
if (m.index > last) tokens.push({ t: "text", v: text.slice(last, m.index) });
|
||||
const closing = m[1] === "/";
|
||||
const name = m[2].trim();
|
||||
if (WRAP_TAG_NAMES.has(name)) tokens.push({ t: closing ? "close" : "open", name });
|
||||
else if (!closing) tokens.push({ t: "point", name }); // stray/unknown closers are ignored
|
||||
last = TAG_RE.lastIndex;
|
||||
}
|
||||
if (last < text.length) tokens.push({ t: "text", v: text.slice(last) });
|
||||
return tokens;
|
||||
}
|
||||
|
||||
export interface DialogueTextNode {
|
||||
t: "text";
|
||||
v: string;
|
||||
}
|
||||
export interface DialoguePointNode {
|
||||
t: "point";
|
||||
name: string;
|
||||
}
|
||||
export interface DialogueSpanNode {
|
||||
t: "span";
|
||||
name: string;
|
||||
children: DialogueNode[];
|
||||
}
|
||||
export type DialogueNode = DialogueTextNode | DialoguePointNode | DialogueSpanNode;
|
||||
|
||||
export function dialogueTree(text: string): DialogueNode[] {
|
||||
const root: { children: DialogueNode[] } = { children: [] };
|
||||
const stack: { children: DialogueNode[] }[] = [root];
|
||||
for (const tk of lexDialogue(text)) {
|
||||
const top = stack[stack.length - 1];
|
||||
if (tk.t === "text") top.children.push({ t: "text", v: tk.v });
|
||||
else if (tk.t === "point") top.children.push({ t: "point", name: tk.name });
|
||||
else if (tk.t === "open") {
|
||||
const node: DialogueSpanNode = { t: "span", name: tk.name, children: [] };
|
||||
top.children.push(node);
|
||||
stack.push(node);
|
||||
} else if (tk.t === "close" && stack.length > 1) stack.pop();
|
||||
}
|
||||
return root.children;
|
||||
}
|
||||
|
||||
/* -------------------------------------------------------------------- */
|
||||
/* Duration estimate */
|
||||
/* -------------------------------------------------------------------- */
|
||||
|
||||
export const SPEECH_RATES = { Measured: 3.2, Natural: 4.0, Quick: 4.8 } as const; // syllables/sec
|
||||
export type SpeechRate = keyof typeof SPEECH_RATES;
|
||||
|
||||
export function syllables(word: string): number {
|
||||
let w = word.toLowerCase().replace(/[^a-z]/g, "");
|
||||
if (!w) return 0;
|
||||
if (w.length <= 3) return 1;
|
||||
w = w.replace(/(?:[^laeiouy]es|ed|[^laeiouy]e)$/, "");
|
||||
w = w.replace(/^y/, "");
|
||||
const m = w.match(/[aeiouy]{1,2}/g);
|
||||
return m ? m.length : 1;
|
||||
}
|
||||
|
||||
export interface DurationEstimate {
|
||||
total: number;
|
||||
speech: number;
|
||||
punct: number;
|
||||
tags: number;
|
||||
syl: number;
|
||||
words: number;
|
||||
low: number;
|
||||
high: number;
|
||||
}
|
||||
|
||||
export function estimateDuration(text: string, rate: number, calib: number): DurationEstimate {
|
||||
const spoken = text.replace(/<\/?[a-zA-Z ]+?>/g, " ");
|
||||
const words = spoken.split(/\s+/).filter(Boolean);
|
||||
const syl = words.reduce((n, w) => n + syllables(w), 0);
|
||||
const speech = syl / rate;
|
||||
|
||||
const sentence = (spoken.match(/[.!?]/g) || []).length * 0.35;
|
||||
const commas = (spoken.match(/[,;:]/g) || []).length * 0.18;
|
||||
const ellipsis = (spoken.match(/\.\.\.|…/g) || []).length * 0.5;
|
||||
const dashes = (spoken.match(/—/g) || []).length * 0.2;
|
||||
const punct = sentence + commas + ellipsis + dashes;
|
||||
|
||||
let tags = 0;
|
||||
const re = /<(\/?)([a-zA-Z ]+?)>/g;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(text))) {
|
||||
if (m[1] === "/") continue;
|
||||
tags += TAG_SECONDS[m[2].trim()] ?? 0;
|
||||
}
|
||||
|
||||
const total = (speech + punct + tags) * calib;
|
||||
return {
|
||||
total,
|
||||
speech: speech * calib,
|
||||
punct: punct * calib,
|
||||
tags: tags * calib,
|
||||
syl,
|
||||
words: words.length,
|
||||
low: total * 0.86,
|
||||
high: total * 1.16,
|
||||
};
|
||||
}
|
||||
|
||||
/** Find the best sentence/pause boundary at which to split text that runs past `ceiling` seconds. */
|
||||
export function suggestSplit(text: string, rate: number, calib: number, ceiling: number): number {
|
||||
const re = /(<\/?[a-zA-Z ]+?>)|([.!?]+)|([,;:]+)|(\s+)|([^\s<.!?,;:]+)/g;
|
||||
let cum = 0;
|
||||
let boundary = 0;
|
||||
let m: RegExpExecArray | null;
|
||||
const budget = ceiling - 1.5;
|
||||
while ((m = re.exec(text))) {
|
||||
if (m[1]) {
|
||||
const closing = m[1].startsWith("</");
|
||||
const name = bareName(m[1]);
|
||||
if (!closing) cum += (TAG_SECONDS[name] ?? 0) * calib;
|
||||
if (name === "pause" || name === "long pause") boundary = re.lastIndex;
|
||||
} else if (m[2]) {
|
||||
cum += 0.35 * calib;
|
||||
boundary = re.lastIndex;
|
||||
} else if (m[3]) {
|
||||
cum += 0.18 * calib;
|
||||
} else if (m[5]) {
|
||||
cum += (syllables(m[5]) / rate) * calib;
|
||||
}
|
||||
if (cum > budget && boundary > 0) return boundary;
|
||||
}
|
||||
return boundary > 0 ? boundary : -1;
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
import { readField, shotTarget, writeField, writeShotTarget } from "@/lib/h3PromptFields";
|
||||
|
||||
const BASE = [
|
||||
"For the target video, <Picture 1> is fully referenced.",
|
||||
"",
|
||||
"integrated_multimodal_description: [Shot 1] A close-up. He (S1) says: <d>[English] Hi.</d> <cutoff>",
|
||||
"",
|
||||
"overall_soundscape: Room tone.",
|
||||
"",
|
||||
"non_diegetic_music: N/A",
|
||||
].join("\n");
|
||||
|
||||
const SECTIONS = [
|
||||
"subject_definitions:",
|
||||
"<Subject 1> is the courier.",
|
||||
"",
|
||||
"retention_analysis:",
|
||||
"<Subject 1> (appears in [Shot 1]): fully_preserved - x.",
|
||||
"",
|
||||
"detailed_description:",
|
||||
"[Shot 1] Wide shot. A rooftop.",
|
||||
"[Shot 2] At 00:02.000, the shot cuts to a close-up shot. A face.",
|
||||
"",
|
||||
"overall_soundscape:",
|
||||
"Rain.",
|
||||
].join("\n");
|
||||
|
||||
describe("readField / writeField", () => {
|
||||
it("reads a field whose text follows the heading on the same line", () => {
|
||||
expect(readField(BASE, "integrated_multimodal_description")).toBe(
|
||||
"[Shot 1] A close-up. He (S1) says: <d>[English] Hi.</d> <cutoff>",
|
||||
);
|
||||
});
|
||||
|
||||
it("reads a field whose text starts on the line after the heading", () => {
|
||||
expect(readField(SECTIONS, "detailed_description")).toBe(
|
||||
"[Shot 1] Wide shot. A rooftop.\n[Shot 2] At 00:02.000, the shot cuts to a close-up shot. A face.",
|
||||
);
|
||||
expect(readField(SECTIONS, "subject_definitions")).toBe("<Subject 1> is the courier.");
|
||||
});
|
||||
|
||||
it("returns null for a field that isn't there", () => {
|
||||
expect(readField(BASE, "detailed_description")).toBeNull();
|
||||
});
|
||||
|
||||
it("rewrites only that field, leaving the rest byte-identical (inline style)", () => {
|
||||
const out = writeField(BASE, "integrated_multimodal_description", "[Shot 1] New.");
|
||||
expect(out).toBe(BASE.replace("[Shot 1] A close-up. He (S1) says: <d>[English] Hi.</d> <cutoff>", "[Shot 1] New."));
|
||||
});
|
||||
|
||||
it("rewrites only that field (own-line style)", () => {
|
||||
const out = writeField(SECTIONS, "detailed_description", "[Shot 1] Only.");
|
||||
expect(out).toContain("detailed_description:\n[Shot 1] Only.\n\noverall_soundscape:\nRain.");
|
||||
expect(out.startsWith("subject_definitions:\n<Subject 1> is the courier.")).toBe(true);
|
||||
});
|
||||
|
||||
it("writing a field's own text back is a no-op", () => {
|
||||
for (const [prompt, field] of [
|
||||
[BASE, "integrated_multimodal_description"],
|
||||
[SECTIONS, "detailed_description"],
|
||||
[SECTIONS, "retention_analysis"],
|
||||
] as const) {
|
||||
expect(writeField(prompt, field, readField(prompt, field) as string)).toBe(prompt);
|
||||
}
|
||||
});
|
||||
|
||||
it("ignores a heading that isn't at the start of a line", () => {
|
||||
expect(readField("see detailed_description: not a header", "detailed_description")).toBeNull();
|
||||
});
|
||||
});
|
||||
|
||||
describe("shotTarget", () => {
|
||||
it("prefers integrated_multimodal_description", () => {
|
||||
expect(shotTarget(BASE).field).toBe("integrated_multimodal_description");
|
||||
});
|
||||
|
||||
it("falls back to detailed_description", () => {
|
||||
expect(shotTarget(SECTIONS).field).toBe("detailed_description");
|
||||
});
|
||||
|
||||
it("treats a plain prompt as the shot text itself", () => {
|
||||
const t = shotTarget("A toy car drives into a plush dog.");
|
||||
expect(t).toEqual({ field: null, body: "A toy car drives into a plush dog." });
|
||||
expect(writeShotTarget("A toy car drives into a plush dog.", t, "[Shot 1] X.")).toBe("[Shot 1] X.");
|
||||
});
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user