From 6b039e80ac377fcc3bc7a2cf91c6b7971ef5a8c5 Mon Sep 17 00:00:00 2001 From: kevin314 Date: Fri, 23 Oct 2026 09:24:59 +0000 Subject: [PATCH] Add scene writer --- apps/fastvideo_studio/mock_server.py | 19 + apps/fastvideo_studio/models/__init__.py | 4 + .../models/scene_draft_request.py | 36 ++ .../fastvideo_studio/scene_writer/__init__.py | 2 + .../scene_writer/dsh/.gitignore | 2 + .../scene_writer/dsh/cordis.patch.yml | 5 + .../scene_writer/dsh/h3-scene-writer.ts | 125 ++++ .../scene_writer/dsh/package.json | 28 + .../scene_writer/dsh/reference-images.spec.ts | 87 +++ .../scene_writer/dsh/reference-images.ts | 131 +++++ .../scene_writer/dsh/validate-job.spec.ts | 128 +++++ .../scene_writer/dsh/validate-job.ts | 224 ++++++++ apps/fastvideo_studio/scene_writer/routes.py | 156 +++++ apps/fastvideo_studio/scene_writer/runner.py | 543 ++++++++++++++++++ .../skills/h3-job-reviser/SKILL.md | 69 +++ .../skills/h3-job-writer/SKILL.md | 190 ++++++ .../skills/h3-job-writer/official-guide.md | 346 +++++++++++ apps/fastvideo_studio/server.py | 5 + apps/fastvideo_studio/src/app/scenes/page.tsx | 33 +- .../components/scenes/ReviseClipDialog.tsx | 230 ++++++++ .../src/components/scenes/SceneBoard.tsx | 17 + .../components/scenes/WriteSceneDialog.tsx | 389 +++++++++++++ apps/fastvideo_studio/src/lib/api.ts | 117 ++++ .../fastvideo_studio/src/lib/lineDiff.test.ts | 26 + apps/fastvideo_studio/src/lib/lineDiff.ts | 38 ++ .../tests/test_scene_writer.py | 126 ++++ 26 files changed, 3075 insertions(+), 1 deletion(-) create mode 100644 apps/fastvideo_studio/models/scene_draft_request.py create mode 100644 apps/fastvideo_studio/scene_writer/__init__.py create mode 100644 apps/fastvideo_studio/scene_writer/dsh/.gitignore create mode 100644 apps/fastvideo_studio/scene_writer/dsh/cordis.patch.yml create mode 100644 apps/fastvideo_studio/scene_writer/dsh/h3-scene-writer.ts create mode 100644 apps/fastvideo_studio/scene_writer/dsh/package.json create mode 100644 apps/fastvideo_studio/scene_writer/dsh/reference-images.spec.ts create mode 100644 apps/fastvideo_studio/scene_writer/dsh/reference-images.ts create mode 100644 apps/fastvideo_studio/scene_writer/dsh/validate-job.spec.ts create mode 100644 apps/fastvideo_studio/scene_writer/dsh/validate-job.ts create mode 100644 apps/fastvideo_studio/scene_writer/routes.py create mode 100644 apps/fastvideo_studio/scene_writer/runner.py create mode 100644 apps/fastvideo_studio/scene_writer/skills/h3-job-reviser/SKILL.md create mode 100644 apps/fastvideo_studio/scene_writer/skills/h3-job-writer/SKILL.md create mode 100644 apps/fastvideo_studio/scene_writer/skills/h3-job-writer/official-guide.md create mode 100644 apps/fastvideo_studio/src/components/scenes/ReviseClipDialog.tsx create mode 100644 apps/fastvideo_studio/src/components/scenes/WriteSceneDialog.tsx create mode 100644 apps/fastvideo_studio/src/lib/lineDiff.test.ts create mode 100644 apps/fastvideo_studio/src/lib/lineDiff.ts create mode 100644 apps/fastvideo_studio/tests/test_scene_writer.py diff --git a/apps/fastvideo_studio/mock_server.py b/apps/fastvideo_studio/mock_server.py index 98c309adf..02402202f 100644 --- a/apps/fastvideo_studio/mock_server.py +++ b/apps/fastvideo_studio/mock_server.py @@ -29,6 +29,7 @@ Usage (from the ``apps/`` directory):: from __future__ import annotations import contextlib +import copy import json import logging import os @@ -50,6 +51,8 @@ from fastapi.responses import FileResponse, PlainTextResponse from fastvideo_studio.database import Database, default_settings_dict from fastvideo_studio.job_queue import circular_dependency, dependencies, plan_dispatch from fastvideo_studio.merge import MergeError, merge_jobs, merged_path +from fastvideo_studio.scene_writer.routes import router as scene_writer_router +from fastvideo_studio.scene_writer.runner import use_server_data_dir from fastvideo_studio.trim import BRIGHTNESS_RANGE, CONTRAST_RANGE, SATURATION_RANGE from fastvideo_studio.models import (CreateDatasetRequest, CreateJobRequest, GradeSegment, MergeSceneRequest, QueueJobsRequest, SettingsUpdate, TrimRequest, UpdateCaptionRequest, @@ -534,6 +537,9 @@ app.add_middleware( allow_methods=["*"], allow_headers=["*"], ) +# The real scene writer: drafting runs the agent against a real model, but +# writes only job files and creates nothing, so it is safe beside mock jobs. +app.include_router(scene_writer_router) @app.get("/api/__mock__") @@ -677,6 +683,17 @@ def create_job(req: CreateJobRequest) -> dict[str, Any]: return _public_job(job) +@app.post("/api/jobs/{job_id}/duplicate", status_code=201) +def duplicate_job(job_id: str) -> dict[str, Any]: + """A new pending job with an existing job's configuration, like the real server's.""" + with _state_lock: + source = _jobs.get(job_id) + if source is None: + raise HTTPException(status_code=404, detail=f"Job {job_id} not found") + config = {k: copy.deepcopy(source[k]) for k in CreateJobRequest.model_fields if k in source} + return create_job(CreateJobRequest(**config)) + + def _start(job: dict[str, Any], at: float | None = None) -> None: # Once the mock is actually driving this job, it's no longer a frozen # historical record -- let _advance_job simulate it like any other. @@ -1201,6 +1218,8 @@ def main() -> None: if args.data_dir is None or not _load_snapshot_from_db(Path(args.data_dir).resolve()): _seed() + if args.data_dir is not None: + use_server_data_dir(Path(args.data_dir).resolve()) uvicorn.run(app, host=args.host, port=args.port, log_level="info") diff --git a/apps/fastvideo_studio/models/__init__.py b/apps/fastvideo_studio/models/__init__.py index 95dde24d9..d9ef7cde2 100644 --- a/apps/fastvideo_studio/models/__init__.py +++ b/apps/fastvideo_studio/models/__init__.py @@ -9,6 +9,7 @@ from fastvideo_studio.models.update_caption_request import UpdateCaptionRequest from fastvideo_studio.models.queue_jobs_request import QueueJobsRequest from fastvideo_studio.models.merge_scene_request import MergeSceneRequest from fastvideo_studio.models.trim_request import GradeSegment, TrimRequest +from fastvideo_studio.models.scene_draft_request import ClipRevisionRequest, SceneDraftReference, SceneDraftRequest def model_label(model_path: str) -> str: @@ -25,5 +26,8 @@ __all__ = [ "MergeSceneRequest", "GradeSegment", "TrimRequest", + "ClipRevisionRequest", + "SceneDraftReference", + "SceneDraftRequest", "model_label", ] diff --git a/apps/fastvideo_studio/models/scene_draft_request.py b/apps/fastvideo_studio/models/scene_draft_request.py new file mode 100644 index 000000000..76fa0d643 --- /dev/null +++ b/apps/fastvideo_studio/models/scene_draft_request.py @@ -0,0 +1,36 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Request model for drafting a scene's jobs with the scene writer.""" + +from typing import Any, Literal + +from pydantic import BaseModel, Field, model_validator + + +class SceneDraftReference(BaseModel): + path: str + media_type: Literal["image", "audio", "video"] + description: str = "" + + +class SceneDraftRequest(BaseModel): + script: str = Field(min_length=1) + scene_name: str = Field(pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$") + references: list[SceneDraftReference] = Field(default_factory=list) + instructions: str = "" + # Let the writer download reference photos from Wikimedia Commons for subjects with none. + find_references: bool = False + + @model_validator(mode="after") + def _needs_references(self) -> "SceneDraftRequest": + if not self.references and not self.find_references: + raise ValueError("Give at least one reference file, or let the writer find references online.") + return self + + +class ClipRevisionRequest(BaseModel): + """Revise one clip from feedback on its generated take; `job` and `neighbors` are Studio jobs.""" + + job: dict[str, Any] + feedback: str = Field(min_length=1) + # The clips before and after it in the scene, for continuity. + neighbors: list[dict[str, Any]] = Field(default_factory=list) diff --git a/apps/fastvideo_studio/scene_writer/__init__.py b/apps/fastvideo_studio/scene_writer/__init__.py new file mode 100644 index 000000000..55c7b0179 --- /dev/null +++ b/apps/fastvideo_studio/scene_writer/__init__.py @@ -0,0 +1,2 @@ +# SPDX-License-Identifier: Apache-2.0 +"""Draft MiniMax-H3 jobs from a script with a DeepSeek Harness agent.""" diff --git a/apps/fastvideo_studio/scene_writer/dsh/.gitignore b/apps/fastvideo_studio/scene_writer/dsh/.gitignore new file mode 100644 index 000000000..1eae0cf67 --- /dev/null +++ b/apps/fastvideo_studio/scene_writer/dsh/.gitignore @@ -0,0 +1,2 @@ +dist/ +node_modules/ diff --git a/apps/fastvideo_studio/scene_writer/dsh/cordis.patch.yml b/apps/fastvideo_studio/scene_writer/dsh/cordis.patch.yml new file mode 100644 index 000000000..05110f028 --- /dev/null +++ b/apps/fastvideo_studio/scene_writer/dsh/cordis.patch.yml @@ -0,0 +1,5 @@ +# Mounts the h3_write_job tool. Each scene-writer run replaces this row's +# config (outDir, references) with an id-targeted patch. +- insert: + - id: h3-scene-writer + name: fastvideo-h3-scene-writer diff --git a/apps/fastvideo_studio/scene_writer/dsh/h3-scene-writer.ts b/apps/fastvideo_studio/scene_writer/dsh/h3-scene-writer.ts new file mode 100644 index 000000000..8e559629c --- /dev/null +++ b/apps/fastvideo_studio/scene_writer/dsh/h3-scene-writer.ts @@ -0,0 +1,125 @@ +/** + * DSH plugin for FastVideo Studio's scene writer. `h3_write_job` validates a + * drafted MiniMax-H3 job against the prompt format and writes it into the + * run's output directory; the agent learns the format from the + * `h3-job-writer` skill, and this tool is the hard check. When the run has a + * reference directory, `find_reference_image` also lets the agent download + * photos from Wikimedia Commons into it for subjects nobody gave a reference + * for; those files then count as allowed references. + */ + +import { mkdir, writeFile } from 'node:fs/promises' +import path from 'node:path' + +import type { Context } from '@deepseek-ai/cordis' +import { defineTool } from '@deepseek-ai/dsh-tools' +import Schema from '@deepseek-ai/schemastery' + +import { fetchReferenceImages } from './reference-images.ts' +import { validateJob } from './validate-job.ts' + +export const name = 'h3-scene-writer' +export const inject = ['tools'] + +export interface Config { + /** Directory the validated job JSONs are written to; set per run. */ + outDir: string + /** Absolute paths of the reference files a job may use. */ + references: string[] + /** The video references among them that carry a soundtrack (each takes an