Add scene writer
This commit is contained in:
kevin314
@@ -29,6 +29,7 @@ Usage (from the ``apps/`` directory)::
|
||||
from __future__ import annotations
|
||||
|
||||
import contextlib
|
||||
import copy
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
@@ -50,6 +51,8 @@ from fastapi.responses import FileResponse, PlainTextResponse
|
||||
from fastvideo_studio.database import Database, default_settings_dict
|
||||
from fastvideo_studio.job_queue import circular_dependency, dependencies, plan_dispatch
|
||||
from fastvideo_studio.merge import MergeError, merge_jobs, merged_path
|
||||
from fastvideo_studio.scene_writer.routes import router as scene_writer_router
|
||||
from fastvideo_studio.scene_writer.runner import use_server_data_dir
|
||||
from fastvideo_studio.trim import BRIGHTNESS_RANGE, CONTRAST_RANGE, SATURATION_RANGE
|
||||
from fastvideo_studio.models import (CreateDatasetRequest, CreateJobRequest, GradeSegment, MergeSceneRequest,
|
||||
QueueJobsRequest, SettingsUpdate, TrimRequest, UpdateCaptionRequest,
|
||||
@@ -534,6 +537,9 @@ app.add_middleware(
|
||||
allow_methods=["*"],
|
||||
allow_headers=["*"],
|
||||
)
|
||||
# The real scene writer: drafting runs the agent against a real model, but
|
||||
# writes only job files and creates nothing, so it is safe beside mock jobs.
|
||||
app.include_router(scene_writer_router)
|
||||
|
||||
|
||||
@app.get("/api/__mock__")
|
||||
@@ -677,6 +683,17 @@ def create_job(req: CreateJobRequest) -> dict[str, Any]:
|
||||
return _public_job(job)
|
||||
|
||||
|
||||
@app.post("/api/jobs/{job_id}/duplicate", status_code=201)
|
||||
def duplicate_job(job_id: str) -> dict[str, Any]:
|
||||
"""A new pending job with an existing job's configuration, like the real server's."""
|
||||
with _state_lock:
|
||||
source = _jobs.get(job_id)
|
||||
if source is None:
|
||||
raise HTTPException(status_code=404, detail=f"Job {job_id} not found")
|
||||
config = {k: copy.deepcopy(source[k]) for k in CreateJobRequest.model_fields if k in source}
|
||||
return create_job(CreateJobRequest(**config))
|
||||
|
||||
|
||||
def _start(job: dict[str, Any], at: float | None = None) -> None:
|
||||
# Once the mock is actually driving this job, it's no longer a frozen
|
||||
# historical record -- let _advance_job simulate it like any other.
|
||||
@@ -1201,6 +1218,8 @@ def main() -> None:
|
||||
|
||||
if args.data_dir is None or not _load_snapshot_from_db(Path(args.data_dir).resolve()):
|
||||
_seed()
|
||||
if args.data_dir is not None:
|
||||
use_server_data_dir(Path(args.data_dir).resolve())
|
||||
|
||||
uvicorn.run(app, host=args.host, port=args.port, log_level="info")
|
||||
|
||||
|
||||
@@ -9,6 +9,7 @@ from fastvideo_studio.models.update_caption_request import UpdateCaptionRequest
|
||||
from fastvideo_studio.models.queue_jobs_request import QueueJobsRequest
|
||||
from fastvideo_studio.models.merge_scene_request import MergeSceneRequest
|
||||
from fastvideo_studio.models.trim_request import GradeSegment, TrimRequest
|
||||
from fastvideo_studio.models.scene_draft_request import ClipRevisionRequest, SceneDraftReference, SceneDraftRequest
|
||||
|
||||
|
||||
def model_label(model_path: str) -> str:
|
||||
@@ -25,5 +26,8 @@ __all__ = [
|
||||
"MergeSceneRequest",
|
||||
"GradeSegment",
|
||||
"TrimRequest",
|
||||
"ClipRevisionRequest",
|
||||
"SceneDraftReference",
|
||||
"SceneDraftRequest",
|
||||
"model_label",
|
||||
]
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Request model for drafting a scene's jobs with the scene writer."""
|
||||
|
||||
from typing import Any, Literal
|
||||
|
||||
from pydantic import BaseModel, Field, model_validator
|
||||
|
||||
|
||||
class SceneDraftReference(BaseModel):
|
||||
path: str
|
||||
media_type: Literal["image", "audio", "video"]
|
||||
description: str = ""
|
||||
|
||||
|
||||
class SceneDraftRequest(BaseModel):
|
||||
script: str = Field(min_length=1)
|
||||
scene_name: str = Field(pattern=r"^[a-z0-9]+(?:-[a-z0-9]+)*$")
|
||||
references: list[SceneDraftReference] = Field(default_factory=list)
|
||||
instructions: str = ""
|
||||
# Let the writer download reference photos from Wikimedia Commons for subjects with none.
|
||||
find_references: bool = False
|
||||
|
||||
@model_validator(mode="after")
|
||||
def _needs_references(self) -> "SceneDraftRequest":
|
||||
if not self.references and not self.find_references:
|
||||
raise ValueError("Give at least one reference file, or let the writer find references online.")
|
||||
return self
|
||||
|
||||
|
||||
class ClipRevisionRequest(BaseModel):
|
||||
"""Revise one clip from feedback on its generated take; `job` and `neighbors` are Studio jobs."""
|
||||
|
||||
job: dict[str, Any]
|
||||
feedback: str = Field(min_length=1)
|
||||
# The clips before and after it in the scene, for continuity.
|
||||
neighbors: list[dict[str, Any]] = Field(default_factory=list)
|
||||
@@ -0,0 +1,2 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Draft MiniMax-H3 jobs from a script with a DeepSeek Harness agent."""
|
||||
@@ -0,0 +1,2 @@
|
||||
dist/
|
||||
node_modules/
|
||||
@@ -0,0 +1,5 @@
|
||||
# Mounts the h3_write_job tool. Each scene-writer run replaces this row's
|
||||
# config (outDir, references) with an id-targeted patch.
|
||||
- insert:
|
||||
- id: h3-scene-writer
|
||||
name: fastvideo-h3-scene-writer
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* DSH plugin for FastVideo Studio's scene writer. `h3_write_job` validates a
|
||||
* drafted MiniMax-H3 job against the prompt format and writes it into the
|
||||
* run's output directory; the agent learns the format from the
|
||||
* `h3-job-writer` skill, and this tool is the hard check. When the run has a
|
||||
* reference directory, `find_reference_image` also lets the agent download
|
||||
* photos from Wikimedia Commons into it for subjects nobody gave a reference
|
||||
* for; those files then count as allowed references.
|
||||
*/
|
||||
|
||||
import { mkdir, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
import type { Context } from '@deepseek-ai/cordis'
|
||||
import { defineTool } from '@deepseek-ai/dsh-tools'
|
||||
import Schema from '@deepseek-ai/schemastery'
|
||||
|
||||
import { fetchReferenceImages } from './reference-images.ts'
|
||||
import { validateJob } from './validate-job.ts'
|
||||
|
||||
export const name = 'h3-scene-writer'
|
||||
export const inject = ['tools']
|
||||
|
||||
export interface Config {
|
||||
/** Directory the validated job JSONs are written to; set per run. */
|
||||
outDir: string
|
||||
/** Absolute paths of the reference files a job may use. */
|
||||
references: string[]
|
||||
/** The video references among them that carry a soundtrack (each takes an <Audio N> label too). */
|
||||
soundVideos: string[]
|
||||
/** Where `find_reference_image` saves downloads; empty leaves the tool out. */
|
||||
refsDir: string
|
||||
/** When set (revising one clip), every written job must keep this name. */
|
||||
expectedName: string
|
||||
}
|
||||
|
||||
export const Config: Schema<Config> = Schema.object({
|
||||
outDir: Schema.string().default(''),
|
||||
references: Schema.array(Schema.string()).default([]),
|
||||
soundVideos: Schema.array(Schema.string()).default([]),
|
||||
refsDir: Schema.string().default(''),
|
||||
expectedName: Schema.string().default(''),
|
||||
})
|
||||
|
||||
export function apply(ctx: Context, config: Config) {
|
||||
const allowed = new Set(config.references)
|
||||
|
||||
ctx.tools.register(defineTool({
|
||||
name: 'h3_write_job',
|
||||
description:
|
||||
'Validate one MiniMax-H3 job (the full job JSON as a string) and, if it passes, write it as <file_name>.json in the output directory. '
|
||||
+ 'Returns the list of problems when it does not pass; fix them and call again.',
|
||||
parameters: {
|
||||
file_name: { type: 'string', required: true, description: 'File stem, e.g. "whiplash_rehearsal_clip_01"' },
|
||||
job_json: { type: 'string', required: true, description: 'The complete job object as JSON text' },
|
||||
},
|
||||
output: {
|
||||
schema: { type: 'string' },
|
||||
render: (_args, value) => [{ type: 'text', text: value }],
|
||||
},
|
||||
async execute(args) {
|
||||
if (!config.outDir) {
|
||||
return 'Error: no output directory is configured for this run; nothing can be written.'
|
||||
}
|
||||
if (!/^[A-Za-z0-9_-]{1,100}$/.test(args.file_name)) {
|
||||
return 'Error: file_name must be letters, digits, "_" or "-" only.'
|
||||
}
|
||||
let job: unknown
|
||||
try {
|
||||
job = JSON.parse(args.job_json)
|
||||
} catch (error) {
|
||||
return `Error: job_json is not valid JSON: ${(error as Error).message}`
|
||||
}
|
||||
const problems = validateJob(job, allowed, new Set(config.soundVideos))
|
||||
if (config.expectedName && (job as { name?: unknown }).name !== config.expectedName) {
|
||||
problems.push(`"name" must stay "${config.expectedName}": a revision is a new take of the same clip.`)
|
||||
}
|
||||
if (problems.length > 0) {
|
||||
return `Not written. Fix these problems and call h3_write_job again:\n- ${problems.join('\n- ')}`
|
||||
}
|
||||
await mkdir(config.outDir, { recursive: true })
|
||||
const file = path.join(config.outDir, `${args.file_name}.json`)
|
||||
await writeFile(file, `${JSON.stringify(job, null, 2)}\n`)
|
||||
return `Wrote ${file}.`
|
||||
},
|
||||
}))
|
||||
|
||||
if (!config.refsDir) return
|
||||
ctx.tools.register(defineTool({
|
||||
name: 'find_reference_image',
|
||||
description:
|
||||
'Search Wikimedia Commons for photos of a person, place, or object that has no reference file, and download up to '
|
||||
+ '`count` candidates as image references. Returns each file\'s path with its title, size, and license. '
|
||||
+ 'Look at each candidate with read_image before using it (Commons results include paintings and group photos). '
|
||||
+ 'Use the chosen paths in "references" with media_type "image".',
|
||||
parameters: {
|
||||
subject: { type: 'string', required: true, description: 'What the reference is for, e.g. "Robert Pattinson" or "jazz rehearsal room"' },
|
||||
query: { type: 'string', required: true, description: 'Search words, e.g. the person\'s full name, or "rehearsal room interior"' },
|
||||
count: { type: 'number', description: 'Candidates to download, 1 to 4 (default 3)' },
|
||||
},
|
||||
output: {
|
||||
schema: { type: 'string' },
|
||||
render: (_args, value) => [{ type: 'text', text: value }],
|
||||
},
|
||||
async execute(args) {
|
||||
const count = Math.min(4, Math.max(1, Math.round(args.count ?? 3)))
|
||||
let images
|
||||
try {
|
||||
await mkdir(config.refsDir, { recursive: true })
|
||||
images = await fetchReferenceImages(config.refsDir, args.subject, args.query, count)
|
||||
} catch (error) {
|
||||
return `Error: ${(error as Error).message}`
|
||||
}
|
||||
if (images.length === 0) {
|
||||
return `No usable photos found on Wikimedia Commons for "${args.query}". Try other words (a full name, a simpler description), or write the job without a reference for this subject.`
|
||||
}
|
||||
for (const image of images) allowed.add(image.path)
|
||||
return [
|
||||
`Downloaded ${images.length} candidate(s) for ${args.subject}:`,
|
||||
...images.map((i) => `- ${i.path} - "${i.title}", ${i.width}x${i.height}, ${i.license}`),
|
||||
'Use one per subject: prefer a clear, recent, front-facing photo of just that subject (titles mentioning "cropped" or "portrait" are often best).',
|
||||
].join('\n')
|
||||
},
|
||||
}))
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
{
|
||||
"name": "fastvideo-h3-scene-writer",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"description": "DSH bundle for FastVideo Studio's scene writer: the h3_write_job and find_reference_image tools.",
|
||||
"type": "module",
|
||||
"main": "./dist/h3-scene-writer.js",
|
||||
"exports": {
|
||||
".": "./dist/h3-scene-writer.js"
|
||||
},
|
||||
"files": [
|
||||
"dist/h3-scene-writer.js",
|
||||
"cordis.patch.yml"
|
||||
],
|
||||
"dsh": {
|
||||
"bundle": {
|
||||
"patch": "./cordis.patch.yml"
|
||||
}
|
||||
},
|
||||
"peerDependencies": {
|
||||
"@deepseek-ai/cordis": "*",
|
||||
"@deepseek-ai/dsh-tools": "*",
|
||||
"@deepseek-ai/schemastery": "*"
|
||||
},
|
||||
"scripts": {
|
||||
"build": "esbuild h3-scene-writer.ts --bundle --external:@deepseek-ai/* --format=esm --platform=node --target=node22 --outfile=dist/h3-scene-writer.js"
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
import { existsSync, mkdtempSync, readFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
import { afterEach, describe, expect, it, vi } from 'vitest'
|
||||
|
||||
import { fetchReferenceImages, slug } from './reference-images.ts'
|
||||
|
||||
function page(index: number, title: string, mime: string, width: number, height: number) {
|
||||
return {
|
||||
index,
|
||||
title: `File:${title}`,
|
||||
imageinfo: [{
|
||||
mime,
|
||||
width,
|
||||
height,
|
||||
thumburl: `https://upload.example/thumb/${encodeURIComponent(title)}`.replace(/\.png$/, '.png'),
|
||||
thumbwidth: 1024,
|
||||
thumbheight: Math.round((1024 * height) / width),
|
||||
descriptionurl: `https://commons.wikimedia.org/wiki/File:${encodeURIComponent(title)}`,
|
||||
extmetadata: {
|
||||
LicenseShortName: { value: 'CC BY-SA 4.0' },
|
||||
Artist: { value: '<a href="//commons.wikimedia.org/wiki/User:Someone">Someone</a>' },
|
||||
},
|
||||
}],
|
||||
}
|
||||
}
|
||||
|
||||
function stubCommons(pages: unknown[]) {
|
||||
const fetchMock = vi.fn(async (url: string) => {
|
||||
if (url.startsWith('https://commons.wikimedia.org/w/api.php')) {
|
||||
return new Response(JSON.stringify({ query: { pages: Object.fromEntries(pages.map((p, i) => [String(i), p])) } }))
|
||||
}
|
||||
return new Response(new Uint8Array([1, 2, 3]))
|
||||
})
|
||||
vi.stubGlobal('fetch', fetchMock)
|
||||
return fetchMock
|
||||
}
|
||||
|
||||
afterEach(() => {
|
||||
vi.unstubAllGlobals()
|
||||
})
|
||||
|
||||
describe('fetchReferenceImages', () => {
|
||||
it('downloads the best photo matches and records their sources', async () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), 'h3-refs-'))
|
||||
stubCommons([
|
||||
page(2, 'Second.jpg', 'image/jpeg', 2000, 3000),
|
||||
page(1, 'First portrait (cropped).jpg', 'image/jpeg', 1200, 1600),
|
||||
page(3, 'Tiny.jpg', 'image/jpeg', 300, 200),
|
||||
page(4, 'Drawing.svg', 'image/svg+xml', 2000, 2000),
|
||||
])
|
||||
|
||||
const images = await fetchReferenceImages(dir, 'Robert Pattinson', 'Robert Pattinson', 3)
|
||||
|
||||
// Search order kept; too-small and non-photo files skipped.
|
||||
expect(images.map((i) => i.title)).toEqual(['First portrait (cropped).jpg', 'Second.jpg'])
|
||||
expect(images.map((i) => path.basename(i.path))).toEqual(['robert_pattinson_1.jpg', 'robert_pattinson_2.jpg'])
|
||||
expect(images.every((i) => existsSync(i.path))).toBe(true)
|
||||
expect(images[0]).toMatchObject({ license: 'CC BY-SA 4.0', artist: 'Someone', width: 1024, height: 1365 })
|
||||
const sources = JSON.parse(readFileSync(path.join(dir, 'sources.json'), 'utf8'))
|
||||
expect(sources).toEqual(images)
|
||||
})
|
||||
|
||||
it('numbers a second search for the same subject after the first', async () => {
|
||||
const dir = mkdtempSync(path.join(tmpdir(), 'h3-refs-'))
|
||||
stubCommons([page(1, 'A.jpg', 'image/jpeg', 800, 800)])
|
||||
await fetchReferenceImages(dir, 'Robert Pattinson', 'Robert Pattinson', 1)
|
||||
const again = await fetchReferenceImages(dir, 'Robert Pattinson', 'Robert Pattinson actor', 1)
|
||||
|
||||
expect(path.basename(again[0].path)).toBe('robert_pattinson_2.jpg')
|
||||
expect(JSON.parse(readFileSync(path.join(dir, 'sources.json'), 'utf8'))).toHaveLength(2)
|
||||
})
|
||||
|
||||
it('reports a failed search', async () => {
|
||||
vi.stubGlobal('fetch', vi.fn(async () => new Response('', { status: 503 })))
|
||||
await expect(fetchReferenceImages(mkdtempSync(path.join(tmpdir(), 'h3-refs-')), 'x', 'x', 1))
|
||||
.rejects.toThrow(/HTTP 503/)
|
||||
})
|
||||
})
|
||||
|
||||
describe('slug', () => {
|
||||
it('makes a safe file stem', () => {
|
||||
expect(slug('Fletcher\'s rehearsal room!')).toBe('fletcher_s_rehearsal_room')
|
||||
expect(slug('***')).toBe('reference')
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,131 @@
|
||||
/**
|
||||
* Reference image search for the scene writer: finds photos on Wikimedia
|
||||
* Commons, downloads them into the run's reference directory, and records
|
||||
* where each came from (page, license, author) in `sources.json` beside them
|
||||
* so the person reviewing the draft can check and credit them.
|
||||
*/
|
||||
|
||||
import { readFile, writeFile } from 'node:fs/promises'
|
||||
import path from 'node:path'
|
||||
|
||||
const COMMONS_API = 'https://commons.wikimedia.org/w/api.php'
|
||||
// Wikimedia asks API clients to identify themselves.
|
||||
const USER_AGENT = 'FastVideoStudio-SceneWriter/0.1 (reference image search)'
|
||||
const DOWNLOAD_WIDTH = 1024
|
||||
const MIN_SIDE = 400
|
||||
const TIMEOUT_MS = 20_000
|
||||
|
||||
export interface FetchedImage {
|
||||
path: string
|
||||
subject: string
|
||||
query: string
|
||||
title: string
|
||||
page: string
|
||||
license: string
|
||||
artist: string
|
||||
width: number
|
||||
height: number
|
||||
}
|
||||
|
||||
interface CommonsPage {
|
||||
index: number
|
||||
title: string
|
||||
imageinfo?: {
|
||||
mime: string
|
||||
width: number
|
||||
height: number
|
||||
thumburl?: string
|
||||
thumbwidth?: number
|
||||
thumbheight?: number
|
||||
descriptionurl: string
|
||||
extmetadata?: Record<string, { value?: string } | undefined>
|
||||
}[]
|
||||
}
|
||||
|
||||
function plainText(html: string | undefined): string {
|
||||
return (html ?? '').replace(/<[^>]*>/g, '').replace(/\s+/g, ' ').trim()
|
||||
}
|
||||
|
||||
export function slug(text: string): string {
|
||||
return text.toLowerCase().replace(/[^a-z0-9]+/g, '_').replace(/^_+|_+$/g, '').slice(0, 40) || 'reference'
|
||||
}
|
||||
|
||||
/** Commons photos (JPEG/PNG, at least MIN_SIDE px on the short side) for a search, best match first. */
|
||||
export async function searchCommons(query: string, limit: number): Promise<CommonsPage[]> {
|
||||
const params = new URLSearchParams({
|
||||
action: 'query',
|
||||
format: 'json',
|
||||
generator: 'search',
|
||||
gsrsearch: `${query} filetype:bitmap`,
|
||||
gsrnamespace: '6',
|
||||
gsrlimit: '20',
|
||||
prop: 'imageinfo',
|
||||
iiprop: 'url|size|mime|extmetadata',
|
||||
iiurlwidth: String(DOWNLOAD_WIDTH),
|
||||
})
|
||||
const response = await fetch(`${COMMONS_API}?${params}`, {
|
||||
headers: { 'user-agent': USER_AGENT },
|
||||
signal: AbortSignal.timeout(TIMEOUT_MS),
|
||||
})
|
||||
if (!response.ok) throw new Error(`Wikimedia Commons search failed: HTTP ${response.status}`)
|
||||
const body = await response.json() as { query?: { pages?: Record<string, CommonsPage> } }
|
||||
return Object.values(body.query?.pages ?? {})
|
||||
.filter((p) => {
|
||||
const info = p.imageinfo?.[0]
|
||||
return info !== undefined && ['image/jpeg', 'image/png'].includes(info.mime)
|
||||
&& Math.min(info.width, info.height) >= MIN_SIDE && info.thumburl !== undefined
|
||||
})
|
||||
.sort((a, b) => a.index - b.index)
|
||||
.slice(0, limit)
|
||||
}
|
||||
|
||||
/**
|
||||
* Search, download up to `count` candidates into `refsDir`, and append them to
|
||||
* `refsDir/sources.json`. Returns the downloaded images.
|
||||
*/
|
||||
export async function fetchReferenceImages(
|
||||
refsDir: string,
|
||||
subject: string,
|
||||
query: string,
|
||||
count: number,
|
||||
): Promise<FetchedImage[]> {
|
||||
const pages = await searchCommons(query, count)
|
||||
const fetched: FetchedImage[] = []
|
||||
const sourcesFile = path.join(refsDir, 'sources.json')
|
||||
let sources: FetchedImage[] = []
|
||||
try {
|
||||
sources = JSON.parse(await readFile(sourcesFile, 'utf8')) as FetchedImage[]
|
||||
} catch (error) {
|
||||
// ENOENT: the first image of this run; anything else would surface on the write below.
|
||||
void error
|
||||
}
|
||||
for (const page of pages) {
|
||||
const info = page.imageinfo![0]
|
||||
const response = await fetch(info.thumburl!, {
|
||||
headers: { 'user-agent': USER_AGENT },
|
||||
signal: AbortSignal.timeout(TIMEOUT_MS),
|
||||
})
|
||||
if (!response.ok) continue
|
||||
const ext = path.extname(new URL(info.thumburl!).pathname).toLowerCase() === '.png' ? '.png' : '.jpg'
|
||||
let file = ''
|
||||
for (let n = 1; ; n++) {
|
||||
file = path.join(refsDir, `${slug(subject)}_${n}${ext}`)
|
||||
if (!sources.some((s) => s.path === file) && !fetched.some((s) => s.path === file)) break
|
||||
}
|
||||
await writeFile(file, Buffer.from(await response.arrayBuffer()))
|
||||
const meta = info.extmetadata ?? {}
|
||||
fetched.push({
|
||||
path: file,
|
||||
subject,
|
||||
query,
|
||||
title: page.title.replace(/^File:/, ''),
|
||||
page: info.descriptionurl,
|
||||
license: plainText(meta.LicenseShortName?.value) || 'unknown license',
|
||||
artist: plainText(meta.Artist?.value) || 'unknown author',
|
||||
width: info.thumbwidth ?? info.width,
|
||||
height: info.thumbheight ?? info.height,
|
||||
})
|
||||
}
|
||||
await writeFile(sourcesFile, `${JSON.stringify([...sources, ...fetched], null, 2)}\n`)
|
||||
return fetched
|
||||
}
|
||||
@@ -0,0 +1,128 @@
|
||||
import { mkdtempSync, writeFileSync } from 'node:fs'
|
||||
import { tmpdir } from 'node:os'
|
||||
import path from 'node:path'
|
||||
|
||||
import { describe, expect, it } from 'vitest'
|
||||
|
||||
import { validateJob } from './validate-job.ts'
|
||||
|
||||
const dir = mkdtempSync(path.join(tmpdir(), 'h3-validate-'))
|
||||
const picture = path.join(dir, 'a.png')
|
||||
const voice = path.join(dir, 'a.mp3')
|
||||
const clip = path.join(dir, 'tail.mp4')
|
||||
writeFileSync(picture, '')
|
||||
writeFileSync(voice, '')
|
||||
writeFileSync(clip, '')
|
||||
|
||||
function goodJob() {
|
||||
return {
|
||||
model_id: 'MiniMaxAI/MiniMax-H3',
|
||||
name: 'scene-clip-01',
|
||||
workload_type: 'i2v',
|
||||
prompt: [
|
||||
'subject_definitions:',
|
||||
'<Subject 1> is Ada from <Picture 1>.',
|
||||
'<Audio 1> is the voice-timbre reference for <Subject 1> (S1), containing a spoken English vocal layer.',
|
||||
'',
|
||||
'summary:',
|
||||
'[reference generation + audio reference] Ada talks.',
|
||||
'',
|
||||
'retention_analysis:',
|
||||
'<Subject 1> (appears in [Shot 1], [Shot 2]): fully_preserved - her face.',
|
||||
'',
|
||||
'detailed_description:',
|
||||
'[Shot 1] Medium shot. (S1) <d>[English] Hello.</d>',
|
||||
'[Shot 2] At 00:03.000, the shot cuts to a close-up shot. (S1) <d>[English] Bye.</d> <cutoff>',
|
||||
'',
|
||||
'overall_soundscape:',
|
||||
'Room tone.',
|
||||
'',
|
||||
'non_diegetic_music:',
|
||||
'N/A',
|
||||
],
|
||||
references: [
|
||||
{ source: picture, media_type: 'image' },
|
||||
{ source: voice, media_type: 'audio' },
|
||||
],
|
||||
negative_prompt: '',
|
||||
guidance_scale: 1,
|
||||
num_frames: 144,
|
||||
fps: 24,
|
||||
height: 768,
|
||||
width: 1344,
|
||||
num_inference_steps: 50,
|
||||
seed: 0,
|
||||
num_gpus: 4,
|
||||
use_fsdp_inference: true,
|
||||
}
|
||||
}
|
||||
|
||||
const allowed = new Set([picture, voice, clip])
|
||||
|
||||
describe('validateJob', () => {
|
||||
it('passes a well-formed job', () => {
|
||||
expect(validateJob(goodJob(), allowed)).toEqual([])
|
||||
})
|
||||
|
||||
it('rejects clips longer than one generation', () => {
|
||||
expect(validateJob({ ...goodJob(), num_frames: 400 }, allowed).join()).toMatch(/num_frames/)
|
||||
})
|
||||
|
||||
it('rejects a tag that points past the reference list', () => {
|
||||
const job = goodJob()
|
||||
job.prompt[1] = '<Subject 1> is Ada from <Picture 2>.'
|
||||
expect(validateJob(job, allowed).join()).toMatch(/<Picture 2> but there are only 1/)
|
||||
})
|
||||
|
||||
it('rejects references that were not provided', () => {
|
||||
const job = goodJob()
|
||||
job.references[0] = { source: '/elsewhere/b.png', media_type: 'image' }
|
||||
expect(validateJob(job, allowed).join()).toMatch(/not one of the reference files/)
|
||||
})
|
||||
|
||||
it('rejects a missing section and cuts past the end of the clip', () => {
|
||||
const job = goodJob()
|
||||
job.prompt = job.prompt.filter((l) => l !== 'overall_soundscape:')
|
||||
job.num_frames = 48
|
||||
const problems = validateJob(job, allowed).join('\n')
|
||||
expect(problems).toMatch(/missing the "overall_soundscape:"/)
|
||||
expect(problems).toMatch(/cuts at 00:03.000, but the clip is only 2.00 s/)
|
||||
})
|
||||
|
||||
it('rejects appearance lists naming shots that do not exist', () => {
|
||||
const job = goodJob()
|
||||
job.prompt[8] = '<Subject 1> (appears in [Shot 1], [Shot 3]): fully_preserved - her face.'
|
||||
expect(validateJob(job, allowed).join()).toMatch(/\[Shot 3\], but there are only 2/)
|
||||
})
|
||||
|
||||
it('requires the audio summary tag to match the references', () => {
|
||||
const job = goodJob()
|
||||
job.prompt[5] = '[reference generation] Ada talks.'
|
||||
expect(validateJob(job, allowed).join()).toMatch(/must include "\+ audio reference\]"/)
|
||||
})
|
||||
|
||||
it('numbers a sound video\'s soundtrack before later audio files', () => {
|
||||
const job = goodJob()
|
||||
job.references = [{ source: clip, media_type: 'video' }, ...job.references]
|
||||
job.prompt[1] = '<Subject 1> is Ada from <Picture 1>, continuing from <Video 1>.'
|
||||
// The video's soundtrack is <Audio 1>, so the voice file is <Audio 2>.
|
||||
expect(validateJob(job, allowed, new Set([clip])).join()).toMatch(/<Audio 2> is in "references" but never described/)
|
||||
job.prompt = job.prompt.map((l) => l.replaceAll('<Audio 1>', '<Audio 2>'))
|
||||
expect(validateJob(job, allowed, new Set([clip]))).toEqual([])
|
||||
// Without a soundtrack the voice file stays <Audio 1>.
|
||||
expect(validateJob(job, allowed).join()).toMatch(/<Audio 2> but there are only 1/)
|
||||
})
|
||||
|
||||
it('enforces the pipeline\'s reference limits', () => {
|
||||
const job = goodJob()
|
||||
job.references = [{ source: voice, media_type: 'audio' }]
|
||||
expect(validateJob(job, allowed).join()).toMatch(/paired with at least one image or video/)
|
||||
job.references = Array.from({ length: 10 }, () => ({ source: picture, media_type: 'image' }))
|
||||
expect(validateJob(job, allowed).join()).toMatch(/at most 9 image references/)
|
||||
})
|
||||
|
||||
it('allows only listed references, and none when the list is empty', () => {
|
||||
expect(validateJob(goodJob(), new Set()).join()).toMatch(/not one of the reference files/)
|
||||
expect(validateJob(goodJob(), null)).toEqual([])
|
||||
})
|
||||
})
|
||||
@@ -0,0 +1,224 @@
|
||||
/**
|
||||
* Structural checks for a drafted MiniMax-H3 job. Each check is something a
|
||||
* model draft has actually gotten wrong, or something that silently breaks a
|
||||
* generation: a missing reference, a tag pointing past the reference list,
|
||||
* appearance lists naming shots that don't exist, cuts after the clip ends.
|
||||
*/
|
||||
|
||||
import { existsSync } from 'node:fs'
|
||||
|
||||
const SECTIONS = [
|
||||
'subject_definitions:',
|
||||
'summary:',
|
||||
'retention_analysis:',
|
||||
'detailed_description:',
|
||||
'overall_soundscape:',
|
||||
'non_diegetic_music:',
|
||||
] as const
|
||||
|
||||
export const MAX_FRAMES = 345
|
||||
export const FPS = 24
|
||||
|
||||
// Ref2VA limits enforced by FastVideo's MiniMax-H3 pipeline (reference.py).
|
||||
const MAX_REFERENCES = { image: 9, video: 3, audio: 3 } as const
|
||||
const MAX_TOTAL_REFERENCES = 12
|
||||
|
||||
const REQUIRED_FIELDS: Record<string, string> = {
|
||||
model_id: 'string',
|
||||
name: 'string',
|
||||
workload_type: 'string',
|
||||
num_frames: 'number',
|
||||
fps: 'number',
|
||||
height: 'number',
|
||||
width: 'number',
|
||||
num_inference_steps: 'number',
|
||||
seed: 'number',
|
||||
num_gpus: 'number',
|
||||
}
|
||||
|
||||
interface Reference {
|
||||
source: string
|
||||
media_type: string
|
||||
}
|
||||
|
||||
function isRecord(value: unknown): value is Record<string, unknown> {
|
||||
return typeof value === 'object' && value !== null && !Array.isArray(value)
|
||||
}
|
||||
|
||||
function highestTag(text: string, kind: string): number {
|
||||
let max = 0
|
||||
for (const m of text.matchAll(new RegExp(`<${kind} (\\d+)>`, 'g'))) max = Math.max(max, Number(m[1]))
|
||||
return max
|
||||
}
|
||||
|
||||
function usedTags(text: string, kind: string): Set<number> {
|
||||
return new Set([...text.matchAll(new RegExp(`<${kind} (\\d+)>`, 'g'))].map((m) => Number(m[1])))
|
||||
}
|
||||
|
||||
function seconds(stamp: string): number {
|
||||
const [mm, ss] = stamp.split(':')
|
||||
return Number(mm) * 60 + Number(ss)
|
||||
}
|
||||
|
||||
/**
|
||||
* Every problem found, as model-facing sentences; empty when the job passes.
|
||||
* `allowedReferences` lists the files a job may use (`null` allows any file that exists).
|
||||
* `soundVideos` are the video references that carry a soundtrack: H3 gives
|
||||
* each of them the next `<Audio N>` label, ahead of its `<Video N>`.
|
||||
*/
|
||||
export function validateJob(
|
||||
job: unknown,
|
||||
allowedReferences: ReadonlySet<string> | null,
|
||||
soundVideos: ReadonlySet<string> = new Set(),
|
||||
): string[] {
|
||||
const problems: string[] = []
|
||||
if (!isRecord(job)) return ['The job must be a JSON object.']
|
||||
|
||||
for (const [field, type] of Object.entries(REQUIRED_FIELDS)) {
|
||||
if (typeof job[field] !== type) problems.push(`"${field}" must be a ${type}.`)
|
||||
}
|
||||
if (job.model_id !== 'MiniMaxAI/MiniMax-H3') problems.push('"model_id" must be "MiniMaxAI/MiniMax-H3".')
|
||||
if (typeof job.name === 'string' && !/-clip-\d{2}$/.test(job.name)) {
|
||||
problems.push('"name" must end in "-clip-NN" (e.g. "whiplash-rehearsal-clip-01") so Studio groups the clips into a scene.')
|
||||
}
|
||||
if (job.fps !== FPS) problems.push(`"fps" must be ${FPS}.`)
|
||||
const frames = typeof job.num_frames === 'number' ? job.num_frames : 0
|
||||
if (!Number.isInteger(frames) || frames < 1 || frames > MAX_FRAMES) {
|
||||
problems.push(`"num_frames" must be a whole number from 1 to ${MAX_FRAMES} (${MAX_FRAMES / FPS} s at ${FPS} fps).`)
|
||||
}
|
||||
const duration = frames / FPS
|
||||
|
||||
// References
|
||||
const refs: Reference[] = []
|
||||
if (!Array.isArray(job.references) || job.references.length === 0) {
|
||||
problems.push('"references" must be a non-empty list of {"source", "media_type"}.')
|
||||
} else {
|
||||
for (const [i, ref] of job.references.entries()) {
|
||||
if (!isRecord(ref) || typeof ref.source !== 'string' || typeof ref.media_type !== 'string') {
|
||||
problems.push(`references[${i}] must be {"source": string, "media_type": string}.`)
|
||||
continue
|
||||
}
|
||||
if (!['image', 'audio', 'video'].includes(ref.media_type)) {
|
||||
problems.push(`references[${i}].media_type must be "image", "audio" or "video".`)
|
||||
}
|
||||
if (allowedReferences !== null && !allowedReferences.has(ref.source)) {
|
||||
problems.push(`references[${i}].source "${ref.source}" is not one of the reference files you were given.`)
|
||||
} else if (!existsSync(ref.source)) {
|
||||
problems.push(`references[${i}].source "${ref.source}" does not exist.`)
|
||||
}
|
||||
refs.push({ source: ref.source, media_type: ref.media_type })
|
||||
}
|
||||
for (const [type, limit] of Object.entries(MAX_REFERENCES)) {
|
||||
const n = refs.filter((r) => r.media_type === type).length
|
||||
if (n > limit) problems.push(`H3 accepts at most ${limit} ${type} references; this job has ${n}.`)
|
||||
}
|
||||
if (refs.length > MAX_TOTAL_REFERENCES) {
|
||||
problems.push(`H3 accepts at most ${MAX_TOTAL_REFERENCES} references in total; this job has ${refs.length}.`)
|
||||
}
|
||||
if (refs.length > 0 && refs.every((r) => r.media_type === 'audio')) {
|
||||
problems.push('Audio references must be paired with at least one image or video reference.')
|
||||
}
|
||||
}
|
||||
|
||||
// Prompt
|
||||
if (!Array.isArray(job.prompt) || !job.prompt.every((line) => typeof line === 'string')) {
|
||||
problems.push('"prompt" must be a list of strings (one line each).')
|
||||
return problems
|
||||
}
|
||||
const lines = job.prompt as string[]
|
||||
const text = lines.join('\n')
|
||||
|
||||
const positions = SECTIONS.map((s) => lines.indexOf(s))
|
||||
SECTIONS.forEach((s, i) => {
|
||||
if (positions[i] === -1) problems.push(`The prompt is missing the "${s}" section header line.`)
|
||||
})
|
||||
if (positions.every((p) => p !== -1) && positions.some((p, i) => i > 0 && p < positions[i - 1])) {
|
||||
problems.push(`The prompt sections must appear in this order: ${SECTIONS.join(', ')}.`)
|
||||
}
|
||||
|
||||
// Tags must match the reference list, numbered the way the pipeline labels
|
||||
// it: one counter per kind in list order, and a video with a soundtrack
|
||||
// takes the next <Audio N> as well as its <Video N>.
|
||||
const counts = { Picture: 0, Audio: 0, Video: 0 }
|
||||
const soundtrackLabels: string[] = []
|
||||
let audioFiles = 0
|
||||
for (const ref of refs) {
|
||||
if (ref.media_type === 'image') counts.Picture++
|
||||
else if (ref.media_type === 'audio') {
|
||||
counts.Audio++
|
||||
audioFiles++
|
||||
} else if (ref.media_type === 'video') {
|
||||
counts.Video++
|
||||
if (soundVideos.has(ref.source)) {
|
||||
counts.Audio++
|
||||
soundtrackLabels.push(`<Audio ${counts.Audio}> (the soundtrack of <Video ${counts.Video}>)`)
|
||||
}
|
||||
}
|
||||
}
|
||||
const soundtrackNumbers = new Set(soundtrackLabels.map((l) => Number(l.match(/^<Audio (\d+)>/)![1])))
|
||||
for (const kind of ['Picture', 'Audio', 'Video'] as const) {
|
||||
const highest = highestTag(text, kind)
|
||||
if (highest > counts[kind]) {
|
||||
problems.push(`The prompt mentions <${kind} ${highest}> but there are only ${counts[kind]} ${kind.toLowerCase()} label(s).`)
|
||||
}
|
||||
const used = usedTags(text, kind)
|
||||
for (let k = 1; k <= counts[kind]; k++) {
|
||||
if (kind === 'Audio' && soundtrackNumbers.has(k)) continue
|
||||
if (!used.has(k)) problems.push(`<${kind} ${k}> is in "references" but never described in the prompt; remove it or describe it.`)
|
||||
}
|
||||
}
|
||||
if (soundtrackLabels.length > 0 && problems.some((p) => p.includes('<Audio'))) {
|
||||
problems.push(`Audio labels in this job: ${soundtrackLabels.join(', ')} come from reference videos with sound, so audio files are numbered after them in list order.`)
|
||||
}
|
||||
|
||||
// Summary tag. A job whose only audio is a video's soundtrack may use either.
|
||||
const summary = positions[1] !== -1 ? lines[positions[1] + 1] ?? '' : ''
|
||||
const hasAudioTag = summary.includes('+ audio reference]')
|
||||
if (summary && audioFiles > 0 && !hasAudioTag) {
|
||||
problems.push('The summary tag must include "+ audio reference]" because the job has audio references.')
|
||||
} else if (summary && counts.Audio === 0 && hasAudioTag) {
|
||||
problems.push('The summary tag must not include "audio reference" because the job has no audio references.')
|
||||
}
|
||||
|
||||
// Shots.
|
||||
const shotLines = lines.filter((l) => /^\[Shot \d+\]/.test(l))
|
||||
if (shotLines.length === 0) problems.push('"detailed_description" needs at least one "[Shot 1] ..." line.')
|
||||
shotLines.forEach((line, i) => {
|
||||
const n = Number(line.match(/^\[Shot (\d+)\]/)![1])
|
||||
if (n !== i + 1) problems.push(`Shots must be numbered 1, 2, 3... in order; found [Shot ${n}] in position ${i + 1}.`)
|
||||
})
|
||||
let lastCut = 0
|
||||
for (const line of shotLines.slice(1)) {
|
||||
const cut = line.match(/^\[Shot (\d+)\] At (\d{2}:\d{2}\.\d{3}), the shot cuts to/)
|
||||
if (!cut) {
|
||||
problems.push(`[Shot ${line.match(/^\[Shot (\d+)\]/)![1]}] must start with "At MM:SS.mmm, the shot cuts to ...".`)
|
||||
continue
|
||||
}
|
||||
const t = seconds(cut[2])
|
||||
if (t <= lastCut) problems.push(`[Shot ${cut[1]}] cuts at ${cut[2]}, which is not after the previous cut.`)
|
||||
if (t >= duration) problems.push(`[Shot ${cut[1]}] cuts at ${cut[2]}, but the clip is only ${duration.toFixed(2)} s long (${frames} frames).`)
|
||||
lastCut = t
|
||||
}
|
||||
|
||||
// Appearance lists must name existing shots.
|
||||
for (const line of lines) {
|
||||
const m = line.match(/^<Subject (\d+)> \(appears in ([^)]*)\)/)
|
||||
if (!m) continue
|
||||
for (const ref of m[2].matchAll(/\[Shot (\d+)\]/g)) {
|
||||
if (Number(ref[1]) > shotLines.length) {
|
||||
problems.push(`<Subject ${m[1]}> is listed as appearing in [Shot ${ref[1]}], but there are only ${shotLines.length} shot(s).`)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Dialogue tags.
|
||||
const opens = (text.match(/<d>/g) ?? []).length
|
||||
const closes = (text.match(/<\/d>/g) ?? []).length
|
||||
if (opens !== closes) problems.push(`Unbalanced dialogue tags: ${opens} "<d>" vs ${closes} "</d>".`)
|
||||
for (const m of text.matchAll(/<d>(?!\[English\] )/g)) {
|
||||
problems.push(`Every dialogue tag must start with "<d>[English] " (found one at character ${m.index}).`)
|
||||
break
|
||||
}
|
||||
|
||||
return problems
|
||||
}
|
||||
@@ -0,0 +1,156 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""API routes for the scene writer: start a draft (or a clip revision), then poll it.
|
||||
|
||||
A draft runs a DeepSeek Harness session for minutes, so each one runs on its
|
||||
own thread and the browser polls `GET /api/scene-writer/drafts/{run_id}`.
|
||||
Drafts never create studio jobs; the browser does that after review.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
from fastapi import APIRouter, HTTPException
|
||||
from fastapi.responses import FileResponse
|
||||
|
||||
from fastvideo_studio.models import ClipRevisionRequest, SceneDraftRequest
|
||||
from fastvideo_studio.scene_writer.runner import (ReferenceFile, SceneWriterSettings, draft_scene, fetched_references,
|
||||
job_file_form, revise_clip)
|
||||
|
||||
logger = logging.getLogger("fastvideo.studio.scene_writer")
|
||||
|
||||
router = APIRouter(prefix="/api/scene-writer")
|
||||
|
||||
_drafts: dict[str, dict[str, Any]] = {}
|
||||
_lock = threading.Lock()
|
||||
|
||||
|
||||
def _run(run_id: str, req: SceneDraftRequest, settings: SceneWriterSettings) -> None:
|
||||
try:
|
||||
result = draft_scene(
|
||||
script=req.script,
|
||||
references=[ReferenceFile(r.path, r.media_type, r.description) for r in req.references],
|
||||
scene_name=req.scene_name,
|
||||
instructions=req.instructions,
|
||||
settings=settings,
|
||||
run_id=run_id,
|
||||
find_references=req.find_references,
|
||||
)
|
||||
update = {
|
||||
"status": "completed",
|
||||
"jobs": result.jobs,
|
||||
"job_files": result.job_files,
|
||||
"final_response": result.final_response,
|
||||
"finish_reason": result.finish_reason,
|
||||
}
|
||||
except Exception as e: # surfaced to the browser; the run directory keeps the details
|
||||
logger.exception("Scene draft %s failed", run_id)
|
||||
update = {"status": "failed", "error": f"{type(e).__name__}: {e}"}
|
||||
with _lock:
|
||||
_drafts[run_id].update(update, finished_at=time.time())
|
||||
|
||||
|
||||
@router.post("/drafts", status_code=202)
|
||||
def start_draft(req: SceneDraftRequest) -> dict[str, Any]:
|
||||
settings = SceneWriterSettings()
|
||||
run_id = uuid.uuid4().hex[:12]
|
||||
with _lock:
|
||||
_drafts[run_id] = {
|
||||
"run_id": run_id,
|
||||
"scene_name": req.scene_name,
|
||||
"status": "running",
|
||||
"started_at": time.time(),
|
||||
"finished_at": None,
|
||||
"jobs": [],
|
||||
"error": None,
|
||||
"run_dir": str(settings.runs_root / run_id),
|
||||
}
|
||||
threading.Thread(target=_run, args=(run_id, req, settings), daemon=True,
|
||||
name=f"scene-draft-{run_id}").start()
|
||||
return get_draft(run_id)
|
||||
|
||||
|
||||
@router.get("/drafts/{run_id}")
|
||||
def get_draft(run_id: str) -> dict[str, Any]:
|
||||
with _lock:
|
||||
draft = _drafts.get(run_id)
|
||||
if draft is None:
|
||||
raise HTTPException(status_code=404, detail="Draft not found")
|
||||
draft = dict(draft)
|
||||
run_dir = Path(draft.pop("run_dir"))
|
||||
if draft["status"] == "running":
|
||||
# Show clips as the agent writes them; each file already passed validation.
|
||||
draft["jobs"] = [json.loads(f.read_text()) for f in sorted((run_dir / "jobs").glob("*.json"))]
|
||||
# Each image is served by name at /drafts/{run_id}/references/{file name}.
|
||||
draft["fetched_references"] = fetched_references(run_dir)
|
||||
return draft
|
||||
|
||||
|
||||
@router.get("/drafts/{run_id}/references/{name}")
|
||||
def get_fetched_reference(run_id: str, name: str) -> FileResponse:
|
||||
"""One image the writer downloaded for a draft, for review in the browser."""
|
||||
with _lock:
|
||||
draft = _drafts.get(run_id)
|
||||
if draft is None:
|
||||
raise HTTPException(status_code=404, detail="Draft not found")
|
||||
refs_dir = Path(draft["run_dir"]) / "refs"
|
||||
if name not in {Path(image["path"]).name for image in fetched_references(refs_dir.parent)}:
|
||||
raise HTTPException(status_code=404, detail="No such fetched reference")
|
||||
return FileResponse(refs_dir / name)
|
||||
|
||||
|
||||
# --- Revisions ----------------------------------------------------------------
|
||||
|
||||
_revisions: dict[str, dict[str, Any]] = {}
|
||||
|
||||
|
||||
def _run_revision(run_id: str, req: ClipRevisionRequest, settings: SceneWriterSettings) -> None:
|
||||
try:
|
||||
result = revise_clip(req.job, req.feedback, req.neighbors, settings=settings, run_id=run_id)
|
||||
if result.job is None:
|
||||
update = {"status": "failed", "final_response": result.final_response,
|
||||
"error": "The reviser finished without writing a revised job."}
|
||||
else:
|
||||
update = {"status": "completed", "job": result.job, "final_response": result.final_response}
|
||||
except Exception as e: # surfaced to the browser; the run directory keeps the details
|
||||
logger.exception("Clip revision %s failed", run_id)
|
||||
update = {"status": "failed", "error": f"{type(e).__name__}: {e}"}
|
||||
with _lock:
|
||||
_revisions[run_id].update(update, finished_at=time.time())
|
||||
|
||||
|
||||
@router.post("/revisions", status_code=202)
|
||||
def start_revision(req: ClipRevisionRequest) -> dict[str, Any]:
|
||||
if not req.job.get("name"):
|
||||
raise HTTPException(status_code=422, detail="The job to revise has no name.")
|
||||
settings = SceneWriterSettings()
|
||||
run_id = uuid.uuid4().hex[:12]
|
||||
with _lock:
|
||||
_revisions[run_id] = {
|
||||
"run_id": run_id,
|
||||
"status": "running",
|
||||
"started_at": time.time(),
|
||||
"finished_at": None,
|
||||
"original": job_file_form(req.job),
|
||||
"job": None,
|
||||
"final_response": None,
|
||||
"error": None,
|
||||
}
|
||||
threading.Thread(target=_run_revision, args=(run_id, req, settings), daemon=True,
|
||||
name=f"clip-revision-{run_id}").start()
|
||||
return get_revision(run_id)
|
||||
|
||||
|
||||
@router.get("/revisions/{run_id}")
|
||||
def get_revision(run_id: str) -> dict[str, Any]:
|
||||
with _lock:
|
||||
revision = _revisions.get(run_id)
|
||||
if revision is None:
|
||||
raise HTTPException(status_code=404, detail="Revision not found")
|
||||
return dict(revision)
|
||||
@@ -0,0 +1,543 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Draft MiniMax-H3 jobs for a scene with a DeepSeek Harness agent.
|
||||
|
||||
One draft is one DSH SDK session in its own workspace:
|
||||
|
||||
<runs_root>/<run_id>/
|
||||
workspace/script.md the scene, as given
|
||||
workspace/references.md the reference files and what they are
|
||||
workspace/.dsh/skills/h3-job-writer/ the job-writing skill (copied)
|
||||
jobs/ the jobs the agent wrote and the validator passed
|
||||
scene-writer.patch.yml points the h3-scene-writer plugin at this run
|
||||
|
||||
The plugin is a DSH bundle (`dsh/`) installed into the harness home's `sdk`
|
||||
profile, so its harness imports resolve; it is (re)installed automatically
|
||||
whenever its files change.
|
||||
|
||||
The agent writes jobs only through the plugin's `h3_write_job` tool, which
|
||||
validates each one, so everything in `jobs/` already passed the structural
|
||||
checks. Nothing is posted to the studio here; the caller shows the drafts.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import threading
|
||||
import uuid
|
||||
from collections.abc import Callable
|
||||
from dataclasses import dataclass, field
|
||||
from pathlib import Path
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
SKILLS_DIR = HERE / "skills"
|
||||
SKILL_DIR = SKILLS_DIR / "h3-job-writer"
|
||||
REVISER_SKILL_DIR = SKILLS_DIR / "h3-job-reviser"
|
||||
BUNDLE_DIR = HERE / "dsh"
|
||||
BUNDLE_SOURCES = ("h3-scene-writer.ts", "validate-job.ts", "reference-images.ts")
|
||||
BUNDLE_FILES = ("package.json", "cordis.patch.yml", *BUNDLE_SOURCES)
|
||||
|
||||
_install_lock = threading.Lock()
|
||||
|
||||
# The harness's own DeepSeek adapter; any other provider goes through its pi-ai
|
||||
# adapter (Cerebras, Groq, OpenAI, ...), which needs a route in the run's patch.
|
||||
DEEPSEEK_PROVIDER = "deepseek-official"
|
||||
DEFAULT_MODELS = {DEEPSEEK_PROVIDER: "deepseek-v4-flash", "cerebras": "qwen-3.8-27b"}
|
||||
# Models a provider serves that the harness's installed catalog does not list yet, as llm-pi-ai model profiles
|
||||
# (from the provider's own model list). `input` must name `image` for the agent's `read_image` tool to work.
|
||||
EXTRA_MODELS = {
|
||||
"cerebras": {
|
||||
"qwen-3.8-27b": {
|
||||
"name": "Qwen 3.8 27B",
|
||||
"contextWindow": 65536,
|
||||
"maxTokens": 32768,
|
||||
"input": ["text", "image"]
|
||||
},
|
||||
},
|
||||
}
|
||||
IMAGE_PIXEL_BUDGET = 1024 * 1024
|
||||
# The child env var the pi-ai route reads its key from.
|
||||
PI_AI_KEY_ENV = "SCENE_WRITER_API_KEY"
|
||||
|
||||
DEFAULT_SETTINGS_FILE = Path.home() / ".config" / "fastvideo-studio" / "scene-writer.env"
|
||||
|
||||
# Drafts hold the reference images their jobs point at, so they belong where
|
||||
# the server that renders those jobs can read them: beside its uploads.
|
||||
_server_runs_root: Path | None = None
|
||||
|
||||
|
||||
def use_server_data_dir(data_dir: Path) -> None:
|
||||
"""Keep drafts under the studio server's data dir unless SCENE_WRITER_RUNS_ROOT says otherwise."""
|
||||
global _server_runs_root
|
||||
_server_runs_root = Path(data_dir) / "scene_writer"
|
||||
|
||||
|
||||
def read_settings_file(path: Path) -> dict[str, str]:
|
||||
"""`KEY=value` lines from the settings file (an optional `export ` prefix and quotes allowed)."""
|
||||
values: dict[str, str] = {}
|
||||
if not path.is_file():
|
||||
return values
|
||||
for line in path.read_text().splitlines():
|
||||
line = line.strip()
|
||||
if not line or line.startswith("#") or "=" not in line:
|
||||
continue
|
||||
key, value = line.removeprefix("export ").split("=", 1)
|
||||
value = value.strip()
|
||||
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
|
||||
value = value[1:-1]
|
||||
values[key.strip()] = value
|
||||
return values
|
||||
|
||||
|
||||
def _setting(name: str, default=None):
|
||||
"""An environment variable, else the settings file, else the default. Read per draft, so file edits apply without a restart."""
|
||||
if name in os.environ:
|
||||
return os.environ[name]
|
||||
path = Path(os.environ.get("SCENE_WRITER_SETTINGS_FILE", DEFAULT_SETTINGS_FILE)).expanduser()
|
||||
return read_settings_file(path).get(name, default)
|
||||
|
||||
|
||||
@dataclass
|
||||
class SceneWriterSettings:
|
||||
"""Where the harness lives and which model drives it (from the environment or the settings file by default)."""
|
||||
|
||||
harness_root: Path = field(
|
||||
default_factory=lambda: Path(_setting("SCENE_WRITER_HARNESS_ROOT", "/mnt/lustre/vlm-kevin/deepseek-harness")))
|
||||
runs_root: Path = field(default_factory=lambda: Path(
|
||||
_setting("SCENE_WRITER_RUNS_ROOT") or _server_runs_root or Path.home() / "h3_scene_writer").expanduser())
|
||||
node_bin_dir: str | None = field(default_factory=lambda: _setting("SCENE_WRITER_NODE_BIN_DIR"))
|
||||
provider: str = field(default_factory=lambda: _setting("SCENE_WRITER_PROVIDER", DEEPSEEK_PROVIDER))
|
||||
model: str | None = field(default_factory=lambda: _setting("SCENE_WRITER_MODEL"))
|
||||
base_url: str | None = field(default_factory=lambda: _setting("SCENE_WRITER_BASE_URL"))
|
||||
api_key: str | None = field(default_factory=lambda: _setting("SCENE_WRITER_API_KEY"))
|
||||
|
||||
@property
|
||||
def dsh_bin(self) -> Path:
|
||||
return self.harness_root / "apps" / "cli" / "lib" / "bin.js"
|
||||
|
||||
@property
|
||||
def sdk_src(self) -> Path:
|
||||
return self.harness_root / "python" / "sdk" / "src"
|
||||
|
||||
@property
|
||||
def dsh_home(self) -> Path:
|
||||
return self.runs_root / "dsh-home"
|
||||
|
||||
@property
|
||||
def uses_pi_ai(self) -> bool:
|
||||
return self.provider != DEEPSEEK_PROVIDER
|
||||
|
||||
def resolved_model(self) -> str:
|
||||
model = self.model or DEFAULT_MODELS.get(self.provider)
|
||||
if not model:
|
||||
raise RuntimeError(f"Set SCENE_WRITER_MODEL: there is no default model for provider {self.provider!r}.")
|
||||
return model
|
||||
|
||||
def child_env(self) -> dict[str, str]:
|
||||
env: dict[str, str] = {}
|
||||
if self.node_bin_dir:
|
||||
env["PATH"] = f"{self.node_bin_dir}{os.pathsep}{os.environ.get('PATH', '')}"
|
||||
if self.uses_pi_ai and self.api_key:
|
||||
env[PI_AI_KEY_ENV] = self.api_key
|
||||
return env
|
||||
|
||||
|
||||
@dataclass
|
||||
class ReferenceFile:
|
||||
path: str
|
||||
media_type: str # "image" | "audio" | "video"
|
||||
description: str = ""
|
||||
|
||||
|
||||
def video_has_sound(path: str) -> bool:
|
||||
"""Whether a reference video carries a soundtrack; H3 then gives it an <Audio N> label too."""
|
||||
try:
|
||||
import av # noqa: PLC0415
|
||||
with av.open(path) as container:
|
||||
return bool(container.streams.audio)
|
||||
except Exception: # noqa: BLE001 - unreadable here means the validator's own existence check reports it
|
||||
return False
|
||||
|
||||
|
||||
def extract_frames(video: str,
|
||||
out_dir: Path,
|
||||
seconds_apart: float = 2.0,
|
||||
max_frames: int = 8,
|
||||
width: int = 768) -> list[Path]:
|
||||
"""Stills from a rendered take for the agent to look at: evenly spaced, about `seconds_apart` apart, each
|
||||
named by its time (`t03.5s.jpg`). Returns nothing for a missing or unreadable video."""
|
||||
try:
|
||||
import av # noqa: PLC0415
|
||||
with av.open(video) as container:
|
||||
stream = container.streams.video[0]
|
||||
frames = [(f.time or 0.0, f.to_image()) for f in container.decode(stream)]
|
||||
except Exception: # noqa: BLE001 - a take without a readable video is revised from its job alone
|
||||
return []
|
||||
if not frames:
|
||||
return []
|
||||
duration = frames[-1][0] + 1 / float(stream.average_rate or 24)
|
||||
count = max(1, min(max_frames, round(duration / seconds_apart)))
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
paths = []
|
||||
for i in range(count):
|
||||
# The middle of each equal slice, so the first and last stills are not the clip's edge frames.
|
||||
t, image = frames[min(len(frames) - 1, int((i + 0.5) * len(frames) / count))]
|
||||
image = image.resize((width, round(image.height * width / image.width)))
|
||||
path = out_dir / f"t{t:04.1f}s.jpg"
|
||||
image.save(path, quality=90)
|
||||
paths.append(path)
|
||||
return paths
|
||||
|
||||
|
||||
@dataclass
|
||||
class DraftResult:
|
||||
run_id: str
|
||||
jobs: list[dict]
|
||||
job_files: list[str]
|
||||
final_response: str | None
|
||||
finish_reason: str | None
|
||||
fetched_references: list[dict] = field(default_factory=list)
|
||||
|
||||
|
||||
def _import_sdk(settings: SceneWriterSettings):
|
||||
if str(settings.sdk_src) not in sys.path:
|
||||
sys.path.insert(0, str(settings.sdk_src))
|
||||
from deepseek_harness import DeepSeekHarness # noqa: PLC0415
|
||||
|
||||
return DeepSeekHarness
|
||||
|
||||
|
||||
def _yaml_str(value: str) -> str:
|
||||
return json.dumps(value) # JSON strings are valid YAML scalars
|
||||
|
||||
|
||||
def _bundle_digest() -> str:
|
||||
digest = hashlib.sha256()
|
||||
for name in BUNDLE_FILES:
|
||||
digest.update(name.encode())
|
||||
digest.update((BUNDLE_DIR / name).read_bytes())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def _build_bundle(settings: SceneWriterSettings, env: dict[str, str]) -> None:
|
||||
"""Compile the plugin into one `dist/` file: DSH loads installed bundles as plain JavaScript."""
|
||||
esbuilds = sorted(
|
||||
(settings.harness_root / "node_modules" / ".pnpm").glob("esbuild@*/node_modules/esbuild/bin/esbuild"))
|
||||
if not esbuilds:
|
||||
raise RuntimeError(f"esbuild not found under {settings.harness_root}/node_modules; run `pnpm install` there.")
|
||||
subprocess.run([
|
||||
str(esbuilds[-1]), BUNDLE_SOURCES[0], "--bundle", "--external:@deepseek-ai/*", "--format=esm",
|
||||
"--platform=node", "--target=node22", "--outfile=dist/h3-scene-writer.js", "--log-level=warning"
|
||||
],
|
||||
cwd=BUNDLE_DIR,
|
||||
env=env,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=120)
|
||||
|
||||
|
||||
def ensure_bundle_installed(settings: SceneWriterSettings) -> None:
|
||||
"""Install the h3-scene-writer bundle into the sdk profile, or refresh a stale copy."""
|
||||
marker = settings.dsh_home / "h3-scene-writer.installed"
|
||||
digest = _bundle_digest()
|
||||
with _install_lock:
|
||||
if marker.is_file() and marker.read_text() == digest:
|
||||
return
|
||||
settings.dsh_home.mkdir(parents=True, exist_ok=True)
|
||||
env = {**os.environ, **settings.child_env(), "DSH_HOME": str(settings.dsh_home)}
|
||||
_build_bundle(settings, env)
|
||||
proc = subprocess.run(
|
||||
["node",
|
||||
str(settings.dsh_bin), "plugin", "--profile", "sdk", "add", "--force", f"file:{BUNDLE_DIR}"],
|
||||
env=env,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=600,
|
||||
check=False)
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(
|
||||
f"Installing the scene-writer bundle failed:\n{proc.stdout[-2000:]}{proc.stderr[-2000:]}")
|
||||
marker.write_text(digest)
|
||||
|
||||
|
||||
def _yaml_list(values: list[str]) -> str:
|
||||
return "".join(f"\n - {_yaml_str(v)}" for v in values) or " []"
|
||||
|
||||
|
||||
def _write_patch(path: Path,
|
||||
out_dir: Path,
|
||||
references: list[ReferenceFile],
|
||||
sound_videos: list[str],
|
||||
refs_dir: Path | None,
|
||||
settings: SceneWriterSettings,
|
||||
expected_name: str = "") -> None:
|
||||
# An id-targeted row replaces the installed row's whole config.
|
||||
text = ("- id: h3-scene-writer\n"
|
||||
" config:\n"
|
||||
f" outDir: {_yaml_str(str(out_dir))}\n"
|
||||
f" references:{_yaml_list([r.path for r in references])}\n"
|
||||
f" soundVideos:{_yaml_list(sound_videos)}\n"
|
||||
f" refsDir: {_yaml_str(str(refs_dir) if refs_dir else '')}\n"
|
||||
f" expectedName: {_yaml_str(expected_name)}\n")
|
||||
if settings.uses_pi_ai:
|
||||
# The base profile mounts llm-pi-ai with no routes; this gives it one for the provider.
|
||||
text += ("- id: llm-pi-ai\n"
|
||||
" config:\n"
|
||||
" providers:\n"
|
||||
f" {_yaml_str(settings.provider)}:\n"
|
||||
f" apiKeyEnv: {PI_AI_KEY_ENV}\n" +
|
||||
(f" baseURL: {_yaml_str(settings.base_url)}\n" if settings.base_url else ""))
|
||||
model = settings.resolved_model()
|
||||
extra = EXTRA_MODELS.get(settings.provider, {}).get(model)
|
||||
if extra:
|
||||
# A `models` list replaces the route's catalog; the run only needs its own model.
|
||||
text += f" models: [{json.dumps({'id': model, **extra})}]\n"
|
||||
# Images the agent reads are sent at most this many pixels (the harness default is 2048x2048): enough to judge
|
||||
# a reference or a take, at a quarter of the tokens, which matters in a 65k context.
|
||||
text += f" requestImagePixelBudget: {IMAGE_PIXEL_BUDGET}\n"
|
||||
path.write_text(text)
|
||||
|
||||
|
||||
def _references_md(references: list[ReferenceFile], sound_videos: list[str], find_references: bool) -> str:
|
||||
|
||||
def row(r: ReferenceFile) -> str:
|
||||
kind = "video with sound: it also takes the next <Audio N> label" if r.path in sound_videos else r.media_type
|
||||
return f"- `{r.path}` ({kind}){': ' + r.description if r.description else ''}"
|
||||
|
||||
rows = "\n".join(row(r) for r in references) if references else "(none given)"
|
||||
text = f"# Reference files\n\nUse these exact absolute paths in `references`.\n\n{rows}\n"
|
||||
if find_references:
|
||||
text += ("\nYou may also use `find_reference_image` to download a photo for a person, place or object the "
|
||||
"script needs that none of these files shows. Never replace a file listed here.\n")
|
||||
return text
|
||||
|
||||
|
||||
def fetched_references(run_dir: Path) -> list[dict]:
|
||||
"""The images `find_reference_image` downloaded in a run, with their source pages and licenses."""
|
||||
sources = run_dir / "refs" / "sources.json"
|
||||
return json.loads(sources.read_text()) if sources.is_file() else []
|
||||
|
||||
|
||||
def build_prompt(scene_name: str, instructions: str) -> str:
|
||||
stem = scene_name.replace("-", "_")
|
||||
extra = f"\n\nAdditional direction from the user:\n{instructions.strip()}" if instructions.strip() else ""
|
||||
return ("/h3-job-writer\n\n"
|
||||
"Write MiniMax-H3 jobs for the scene in `script.md`, using only the reference files listed in "
|
||||
"`references.md` (and any it lets you download). "
|
||||
f"Name the jobs `{scene_name}-clip-01`, `{scene_name}-clip-02`, ... and pass `file_name` values "
|
||||
f"`{stem}_clip_01`, `{stem}_clip_02`, ... to `h3_write_job`. "
|
||||
"Write every clip through `h3_write_job` and fix anything it reports. Do not post or start jobs."
|
||||
f"{extra}")
|
||||
|
||||
|
||||
def _check_ready(settings: SceneWriterSettings) -> str:
|
||||
"""The model to use; raises when the harness or the model settings are not usable."""
|
||||
model = settings.resolved_model()
|
||||
if settings.uses_pi_ai and not settings.api_key:
|
||||
raise RuntimeError(f"Set SCENE_WRITER_API_KEY to your {settings.provider} API key.")
|
||||
if not settings.dsh_bin.is_file():
|
||||
raise RuntimeError(f"DeepSeek Harness is not built: {settings.dsh_bin} is missing "
|
||||
"(run `pnpm install && pnpm run build` in the harness checkout).")
|
||||
ensure_bundle_installed(settings)
|
||||
return model
|
||||
|
||||
|
||||
def _run_session(settings: SceneWriterSettings, model: str, workspace: Path, patch: Path, prompt: str, session_id: str,
|
||||
on_event: Callable[[dict], None] | None):
|
||||
"""Run one agent turn in `workspace` with the run's patch; returns the SDK's RunResult."""
|
||||
DeepSeekHarness = _import_sdk(settings)
|
||||
with DeepSeekHarness(
|
||||
dsh_home=str(settings.dsh_home),
|
||||
cwd=str(workspace),
|
||||
dsh_bin=str(settings.dsh_bin),
|
||||
profile="sdk",
|
||||
patches=(str(patch), ),
|
||||
provider=settings.provider,
|
||||
model=model,
|
||||
# These two reach only the DeepSeek adapter; a pi-ai route gets them from the patch.
|
||||
base_url=None if settings.uses_pi_ai else settings.base_url,
|
||||
api_key=None if settings.uses_pi_ai else settings.api_key,
|
||||
env=settings.child_env(),
|
||||
) as harness:
|
||||
result = harness.run(
|
||||
prompt,
|
||||
session_id=session_id,
|
||||
on_notification=(lambda n: on_event(n.payload)) if on_event else None,
|
||||
)
|
||||
# A plugin that fails to load only logs to stderr and the session runs without its tools.
|
||||
load_errors = [
|
||||
line for line in list(harness.client._stderr_lines)
|
||||
if line.startswith("h3-scene-writer (fastvideo-h3-scene-writer):")
|
||||
]
|
||||
if load_errors:
|
||||
raise RuntimeError(f"The scene-writer plugin failed to load: {load_errors[0]}")
|
||||
if result.finish_reason == "error":
|
||||
raise RuntimeError(_turn_error_message(result.events, settings))
|
||||
return result
|
||||
|
||||
|
||||
def _turn_error_message(events: list[dict], settings: SceneWriterSettings) -> str:
|
||||
"""Explain why the agent's turn ended in an error (usually the model provider refusing the request)."""
|
||||
message = "unknown error"
|
||||
for event in reversed(events):
|
||||
if event.get("type") == "turn/end":
|
||||
error = ((event.get("data") or {}).get("reason") or {}).get("error") or {}
|
||||
message = error.get("message") or message
|
||||
break
|
||||
if message.startswith("402"):
|
||||
return (f"{settings.provider} refused the request with 402 Payment Required: the account "
|
||||
"behind SCENE_WRITER_API_KEY has no credits or billing set up.")
|
||||
if message.startswith("401") or message.startswith("403"):
|
||||
return f"{settings.provider} rejected SCENE_WRITER_API_KEY ({message})."
|
||||
return f"The model call failed: {message}"
|
||||
|
||||
|
||||
def draft_scene(
|
||||
script: str,
|
||||
references: list[ReferenceFile],
|
||||
scene_name: str,
|
||||
instructions: str = "",
|
||||
settings: SceneWriterSettings | None = None,
|
||||
on_event: Callable[[dict], None] | None = None,
|
||||
run_id: str | None = None,
|
||||
find_references: bool = False,
|
||||
) -> DraftResult:
|
||||
"""Run one drafting session and return the jobs it wrote, in clip order.
|
||||
|
||||
With `find_references`, the agent may download reference photos from Wikimedia Commons into the run's
|
||||
`refs/` directory; `fetched_references` lists them.
|
||||
"""
|
||||
if not references and not find_references:
|
||||
raise ValueError("Give at least one reference file, or let the writer find references online.")
|
||||
settings = settings or SceneWriterSettings()
|
||||
model = _check_ready(settings)
|
||||
run_id = run_id or uuid.uuid4().hex[:12]
|
||||
run_dir = settings.runs_root / run_id
|
||||
workspace = run_dir / "workspace"
|
||||
out_dir = run_dir / "jobs"
|
||||
workspace.mkdir(parents=True, exist_ok=False)
|
||||
out_dir.mkdir()
|
||||
(workspace / "script.md").write_text(script)
|
||||
sound_videos = [r.path for r in references if r.media_type == "video" and video_has_sound(r.path)]
|
||||
(workspace / "references.md").write_text(_references_md(references, sound_videos, find_references))
|
||||
shutil.copytree(SKILL_DIR, workspace / ".dsh" / "skills" / SKILL_DIR.name)
|
||||
patch = run_dir / "scene-writer.patch.yml"
|
||||
_write_patch(patch, out_dir, references, sound_videos, run_dir / "refs" if find_references else None, settings)
|
||||
|
||||
result = _run_session(settings, model, workspace, patch, build_prompt(scene_name, instructions), f"scene-{run_id}",
|
||||
on_event)
|
||||
files = sorted(out_dir.glob("*.json"))
|
||||
return DraftResult(
|
||||
run_id=run_id,
|
||||
jobs=[json.loads(f.read_text()) for f in files],
|
||||
job_files=[str(f) for f in files],
|
||||
final_response=result.final_response,
|
||||
finish_reason=result.finish_reason,
|
||||
fetched_references=fetched_references(run_dir),
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class RevisionResult:
|
||||
run_id: str
|
||||
original: dict
|
||||
#: the revised job, or None when the agent wrote nothing
|
||||
job: dict | None
|
||||
#: the agent's explanation of the cause and its changes
|
||||
final_response: str | None
|
||||
finish_reason: str | None
|
||||
|
||||
|
||||
# The job-file fields; a Studio job also carries runtime state (status, outputs) the agent does not need.
|
||||
JOB_FILE_FIELDS = ("model_id", "name", "workload_type", "prompt", "references", "negative_prompt", "guidance_scale",
|
||||
"num_frames", "fps", "height", "width", "num_inference_steps", "seed", "num_gpus",
|
||||
"use_fsdp_inference")
|
||||
|
||||
|
||||
def job_file_form(job: dict) -> dict:
|
||||
"""A job in the job-file form the skills use: only job-file fields, `prompt` as a list of lines."""
|
||||
form = {k: job[k] for k in JOB_FILE_FIELDS if job.get(k) is not None}
|
||||
prompt = form.get("prompt", "")
|
||||
form["prompt"] = prompt.split("\n") if isinstance(prompt, str) else list(prompt)
|
||||
form["references"] = [{"source": r["source"], "media_type": r["media_type"]} for r in form.get("references") or []]
|
||||
return form
|
||||
|
||||
|
||||
def build_revision_prompt(job_name: str, file_name: str, has_frames: bool = False) -> str:
|
||||
frames = ("Stills from the rendered take are in `take_frames/`, named by their time in the clip; look at them "
|
||||
"with `read_image` and check the feedback against what was rendered. " if has_frames else "")
|
||||
return ("/h3-job-reviser\n\n"
|
||||
f"Revise the clip `{job_name}` in `current_job.json` to fix the problem described in `feedback.md`. "
|
||||
"The clips around it in the scene are in `neighbors/`. "
|
||||
f"{frames}"
|
||||
f"Write the revised job with `h3_write_job`, keeping the name `{job_name}` and passing `file_name` "
|
||||
f"`{file_name}`. Do not post or start jobs. Finish with the cause you found and each change you made.")
|
||||
|
||||
|
||||
def revise_clip(
|
||||
job: dict,
|
||||
feedback: str,
|
||||
neighbors: list[dict] | None = None,
|
||||
settings: SceneWriterSettings | None = None,
|
||||
on_event: Callable[[dict], None] | None = None,
|
||||
run_id: str | None = None,
|
||||
) -> RevisionResult:
|
||||
"""Run one revision session for a clip job (job-file or Studio form) and return the revised job.
|
||||
|
||||
The revision may use the clip's own references and its neighbours' (for example to add a voice that the
|
||||
clip before it already uses); it must keep the clip's name.
|
||||
"""
|
||||
name = job.get("name") or ""
|
||||
if not name:
|
||||
raise ValueError("The job to revise has no name.")
|
||||
if not feedback.strip():
|
||||
raise ValueError("Describe what is wrong with the clip.")
|
||||
settings = settings or SceneWriterSettings()
|
||||
model = _check_ready(settings)
|
||||
run_id = run_id or uuid.uuid4().hex[:12]
|
||||
run_dir = settings.runs_root / run_id
|
||||
workspace = run_dir / "workspace"
|
||||
out_dir = run_dir / "jobs"
|
||||
(workspace / "neighbors").mkdir(parents=True, exist_ok=False)
|
||||
out_dir.mkdir()
|
||||
|
||||
original = job_file_form(job)
|
||||
neighbors = [job_file_form(n) for n in neighbors or []]
|
||||
(workspace / "current_job.json").write_text(json.dumps(original, indent=2) + "\n")
|
||||
(workspace / "feedback.md").write_text(f"# What went wrong in {name}\n\n{feedback.strip()}\n")
|
||||
for n in neighbors:
|
||||
stem = (n.get("name") or "clip").replace("-", "_")
|
||||
(workspace / "neighbors" / f"{stem}.json").write_text(json.dumps(n, indent=2) + "\n")
|
||||
|
||||
references: list[ReferenceFile] = []
|
||||
for j in [original, *neighbors]:
|
||||
for r in j.get("references") or []:
|
||||
if r["source"] not in {ref.path for ref in references}:
|
||||
references.append(ReferenceFile(r["source"], r["media_type"]))
|
||||
sound_videos = [r.path for r in references if r.media_type == "video" and video_has_sound(r.path)]
|
||||
(workspace / "references.md").write_text(_references_md(references, sound_videos, False))
|
||||
for skill in (SKILL_DIR, REVISER_SKILL_DIR):
|
||||
shutil.copytree(skill, workspace / ".dsh" / "skills" / skill.name)
|
||||
patch = run_dir / "scene-writer.patch.yml"
|
||||
_write_patch(patch, out_dir, references, sound_videos, None, settings, expected_name=name)
|
||||
|
||||
# A Studio take carries its rendered video; a job file has none.
|
||||
frames = extract_frames(job["output_path"], workspace / "take_frames") if job.get("output_path") else []
|
||||
|
||||
file_name = name.replace("-", "_")
|
||||
result = _run_session(settings, model, workspace, patch, build_revision_prompt(name, file_name, bool(frames)),
|
||||
f"revise-{run_id}", on_event)
|
||||
# The newest file the agent wrote; the plugin already checked it keeps the clip's name.
|
||||
written = sorted(out_dir.glob("*.json"), key=lambda f: f.stat().st_mtime)
|
||||
return RevisionResult(
|
||||
run_id=run_id,
|
||||
original=original,
|
||||
job=json.loads(written[-1].read_text()) if written else None,
|
||||
final_response=result.final_response,
|
||||
finish_reason=result.finish_reason,
|
||||
)
|
||||
@@ -0,0 +1,69 @@
|
||||
---
|
||||
name: h3-job-reviser
|
||||
description: Revise one MiniMax-H3 clip job from a person's feedback on its generated video, making the smallest prompt change that fixes the problem without breaking the clip's continuity with its neighbours.
|
||||
---
|
||||
|
||||
# Revising a MiniMax-H3 clip from feedback
|
||||
|
||||
A person generated a clip, watched it, and described what went wrong. You rewrite that clip's job
|
||||
so the next take fixes it. Work from the feedback, the job, what these rules say the model does,
|
||||
and, when `take_frames/` exists, stills from the take itself.
|
||||
|
||||
The job format, the section order, the numbering rules and the **Rules learned from real
|
||||
generations** are in `.dsh/skills/h3-job-writer/SKILL.md`. Read that file first; everything in it
|
||||
applies here.
|
||||
|
||||
## Inputs in the workspace
|
||||
|
||||
- `current_job.json`: the job as it was generated.
|
||||
- `feedback.md`: what went wrong, in the person's words.
|
||||
- `neighbors/`: the clips before and after it in the scene, when there are any. Positions,
|
||||
wardrobe, who is in the room and what the music is doing must stay consistent with them.
|
||||
- `references.md`: the reference files this job may use.
|
||||
- `take_frames/` (when the take's video was available): stills from the rendered take, about two
|
||||
seconds apart, named by their time in the clip (`t03.5s.jpg` is 3.5 s in).
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Read the writer skill, then the job, the feedback, and the neighbours. Look at every still in
|
||||
`take_frames/` and at the clip's image references with `read_image`. Use the stills to find
|
||||
where in the clip the problem starts and what the model actually rendered (an extra person,
|
||||
the wrong seat, a reference's background copied in, the wrong wardrobe). Stills are sparse and
|
||||
silent: for counts of fast repeated actions, timing within two seconds, and anything heard,
|
||||
trust the feedback over the stills. If the stills contradict the feedback, say so in your
|
||||
explanation and fix what the feedback describes.
|
||||
2. Diagnose before editing. Name the cause in the prompt, not just the symptom (table below).
|
||||
3. Make the smallest change that removes the cause. Keep everything that the feedback does not
|
||||
mention: wording that worked is worth keeping, and every change can create a new problem.
|
||||
4. Re-read the whole prompt for text the change made stale (rule 9 of the writer skill): the
|
||||
summary, the `appears in` lists, the soundscape, and any sentence that still describes the old
|
||||
behaviour. Fix those too.
|
||||
5. Write the job with `h3_write_job`, keeping its `name` and using the `file_name` you were given.
|
||||
Fix anything it reports.
|
||||
6. End with a short explanation for the person: the cause you found, each change you made, and
|
||||
anything they should watch for in the next take. If the feedback asks for something the
|
||||
format cannot promise, say so.
|
||||
|
||||
## Common symptoms and their usual causes
|
||||
|
||||
| Feedback | Usual cause | Fix |
|
||||
| --- | --- | --- |
|
||||
| An action happens too many times ("3 slaps instead of 2") | The prompt describes it as a repeated pattern ("on each four"), so the model keeps the rhythm going; or the clip has time left after the last scripted beat | Describe each occurrence once, tied to a word, with no "each"/"every"; remove counts like "exactly two"; end the clip shortly after the last one (shorten `num_frames`) |
|
||||
| A pause or stall at a line break | Speech split across several `<d>` tags with prose between them | Merge the run of speech into one `<d>` and put the actions after it |
|
||||
| Speech too slow or too fast | Ellipses, spelled-out pauses, or no tempo stated | Dashes for fast speech, no ellipses; state the tempo |
|
||||
| A character appears twice, or in the wrong place | The shot does not say where they are | Pin every character's position in every shot; add a whole-clip rule to the summary ("there is only one drummer") |
|
||||
| Someone moves too early | Their position is not pinned in the earlier shots, or the summary implies the move | Pin them in place in each shot before the move; move them only in the shot where the script does |
|
||||
| Background people vanish | A reference photo of an empty room, and the shot does not mention them | Name the background people in every shot, and say the photo shows the room before they arrived |
|
||||
| Music restarts or keeps playing | An audio reference plays from its start in every clip that uses it; the stop is not stated everywhere | Drop the music reference from clips after it ends; state when it stops in the summary, the shot and the soundscape |
|
||||
| Delivery is louder, angrier or more emphatic than wanted | Added direction ("roars", "shouts", all caps) | Remove it; keep the script's wording |
|
||||
| Something unwanted fills the end of the clip | The clip is longer than its scripted content | Shorten `num_frames` to end soon after the last beat |
|
||||
|
||||
When the feedback describes something new, reason the same way: what in the prompt (or missing
|
||||
from it) would make the model do that?
|
||||
|
||||
## Do not
|
||||
|
||||
- Do not change the clip's `name`, its references' order (labels like `<Picture 2>` depend on
|
||||
it), the resolution, or the seed, unless the feedback is about them.
|
||||
- Do not rewrite sections the feedback does not touch "while you are there".
|
||||
- Do not add a fix for a problem nobody reported.
|
||||
@@ -0,0 +1,190 @@
|
||||
---
|
||||
name: h3-job-writer
|
||||
description: Turn a screenplay scene plus reference images/audio into MiniMax-H3 Ref2VA job JSONs for FastVideo Studio, one job per clip, following the H3 prompt format and the timing/staging rules learned from real generations.
|
||||
---
|
||||
|
||||
# Writing MiniMax-H3 jobs from a script
|
||||
|
||||
You turn one scene of a script into a sequence of FastVideo Studio jobs, one per clip.
|
||||
Every clip is generated independently, so each job's prompt must stand on its own.
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Read the script and the reference list you were given. Note every character, the setting,
|
||||
every action, and every line of dialogue in order. For the full prompt format with a complete
|
||||
example, read `.dsh/skills/h3-job-writer/official-guide.md` (MiniMax's own guide). Where it
|
||||
disagrees with this file, this file wins.
|
||||
Look at every image reference with `read_image` (see **Looking at references**).
|
||||
2. Plan the clips (see **Splitting into clips**). Write the plan down before drafting.
|
||||
3. Draft each clip as a job object and call `h3_write_job` for it. The tool validates the job
|
||||
and writes it; fix every error it reports and call it again until it succeeds.
|
||||
4. Finish with a short summary listing each clip: its file, length, and what happens in it.
|
||||
|
||||
Never post, queue, or start jobs. A human reviews every draft first.
|
||||
|
||||
## Job JSON
|
||||
|
||||
```json
|
||||
{
|
||||
"model_id": "MiniMaxAI/MiniMax-H3",
|
||||
"name": "<scene>-clip-01",
|
||||
"workload_type": "i2v",
|
||||
"prompt": ["subject_definitions:", "...", "", "summary:", "...", ...],
|
||||
"references": [
|
||||
{"source": "/abs/path/fletcher_ref.png", "media_type": "image"},
|
||||
{"source": "/abs/path/fletcher.mp3", "media_type": "audio"}
|
||||
],
|
||||
"negative_prompt": "",
|
||||
"guidance_scale": 1,
|
||||
"num_frames": 288,
|
||||
"fps": 24,
|
||||
"height": 768,
|
||||
"width": 1344,
|
||||
"num_inference_steps": 50,
|
||||
"seed": 0,
|
||||
"num_gpus": 4,
|
||||
"use_fsdp_inference": true
|
||||
}
|
||||
```
|
||||
|
||||
- `prompt` is a list of lines; they are joined with newlines before posting.
|
||||
- `num_frames` = clip seconds × 24, at most **345** (14.4 s). Size the clip to its scripted content (see rule 5).
|
||||
- Clip names share one stem and end in `-clip-NN` so Studio groups them into one scene.
|
||||
- Only use reference paths you were given. Include only the references a clip actually uses.
|
||||
- At most 9 images, 3 videos and 3 audio files per job, 12 references in total, and at least one
|
||||
image or video (audio alone is refused).
|
||||
|
||||
## Prompt sections (in this order)
|
||||
|
||||
```
|
||||
subject_definitions:
|
||||
<Subject 1> is <who> from <Picture 1>: <appearance, wardrobe, demeanor>.
|
||||
<Subject 2> is ...
|
||||
<Subject N> is the <location> from <Picture N>: <layout, fixtures>. It also sets the lighting for the whole scene: <light>. <What the picture is / is not, see rule 7>.
|
||||
<Audio 1> is the voice-timbre reference for <Subject 1> (S1), containing a spoken English vocal layer.
|
||||
<Video 1> is ... (only when continuing from a previous clip's tail)
|
||||
|
||||
summary:
|
||||
[reference generation + audio reference] The target video shows ... <global rules: who stays where, who never does what>. The setting is the <location> from <Subject N>, and the whole scene shares its <light>.
|
||||
|
||||
retention_analysis:
|
||||
<Subject 1> (appears in [Shot 1], [Shot 2]): fully_preserved - <what is kept>; his own background and lighting are not, and he is lit by the <location>'s light.
|
||||
<Subject N> (appears in [Shot 1], [Shot 2]): partially_preserved - <setting elements kept>; only the setting is retained, with no camera framing carried over.
|
||||
<Audio 1>: reference - its vocal timbre guides the dialogue delivery of <Subject 1> without copying the original signal.
|
||||
|
||||
detailed_description:
|
||||
[Shot 1] <Framing>. The camera is fixed and does not move. <Who is where>. <Action>. <Subject 1> (S1) speaks in the voice timbre referenced from <Audio 1>. (S1) <d>[English] Line.</d>
|
||||
[Shot 2] At 00:04.500, the shot cuts to <framing>. ...
|
||||
|
||||
overall_soundscape:
|
||||
<diegetic sound, with what must NOT be heard>
|
||||
|
||||
non_diegetic_music:
|
||||
N/A
|
||||
```
|
||||
|
||||
Numbering: `<Picture k>`, `<Video k>` and `<Audio k>` each count their own kind, in the order the
|
||||
references appear in `references`. A **video with sound** (marked in `references.md`) takes the
|
||||
next `<Audio k>` as well as its `<Video k>`, even if the prompt never mentions that soundtrack, so
|
||||
every audio file after it shifts up by one. Example: `[video with sound, voice.mp3, face.png]` are
|
||||
`<Video 1>` + `<Audio 1>`, then `<Audio 2>`, then `<Picture 1>`. (The official guide says a video
|
||||
does not create an audio label just because it has sound; FastVideo's pipeline does, and this rule
|
||||
follows the pipeline.) Subjects are numbered in definition order. The tag in the
|
||||
summary is `[reference generation + audio reference]` when any audio reference is used, else
|
||||
`[reference generation]`; add `video continuation + ` in front when a `<Video 1>` tail is used.
|
||||
|
||||
Dialogue: `(S1) <d>[English] text</d>`. Add `<cutoff>` after a line that is interrupted.
|
||||
Speaker IDs (S1, S2...) match the subject numbers of the speakers.
|
||||
|
||||
The `appears in` lists in `retention_analysis` must name exactly the shots where that subject is
|
||||
on screen. Keep them in sync whenever shots change.
|
||||
|
||||
## Rules learned from real generations
|
||||
|
||||
These are the reasons previous drafts failed. Follow all of them.
|
||||
|
||||
1. **Repeated actions become a rhythm the model continues.** "Slap on each four" turned into a
|
||||
steady beat that kept going. Keep a repeated action to the fewest repetitions the script needs,
|
||||
never write exceptions inside a pattern ("the third one gets no slap"), and end the pattern by
|
||||
ending the clip or cutting to a new shot. Counts like "exactly two" do not stop a rhythm.
|
||||
2. **Keep rhythmic speech in one dialogue tag.** Splitting a count into several `<d>` tags with
|
||||
prose between them makes the speaker pause at every break. Write one continuous line
|
||||
(`One-two-three-four-one-two-three-four-`) and describe actions that land on it in prose after it.
|
||||
3. **Write speech the way it should sound.** Ellipses slow delivery ("One... two..."); use dashes
|
||||
for fast speech. Spell words out ("four", not "fo-"). State tempo explicitly when it matters
|
||||
("150 BPM, 0.4 seconds per number, never pausing").
|
||||
4. **Timestamps are loose guides.** Use them for shot cuts and rough placement. Do not rely on a
|
||||
timestamp to sync an action to a word: anchor the action to the word itself ("as he says
|
||||
'four'"). Never pin an action to a time that the speech could drift away from.
|
||||
5. **Every second of the clip must be scripted.** Spare time at the end gets filled with more of
|
||||
whatever was happening (more slaps, the music starting over). End the clip shortly after its
|
||||
last scripted beat: estimate speech at ~2.5 words/second (faster for counting at the stated
|
||||
tempo) plus the time each action needs.
|
||||
6. **Pin every character's position in every shot.** If a shot does not say where someone is, the
|
||||
model invents it (a second copy of a character, someone walking over early). Put a whole-clip
|
||||
rule in the summary ("X stays seated behind his drum kit for the entire video; he never stands
|
||||
up", "there is only one drummer"), and restate positions in each shot. Use separate singles,
|
||||
not two-shots, when characters must stay apart. Frame anyone whose hands matter fully in shot;
|
||||
a half-framed actor's gestures read as extra actions.
|
||||
7. **Reference images win in wide shots.** An empty location photo produces an empty room. Say
|
||||
what the picture is ("This picture shows the room before the band arrives") and name the
|
||||
background people in every shot ("Every chair is filled: the band musicians sit at their
|
||||
stands..."), not just once in the subject definition.
|
||||
8. **Audio references play from their start.** A music reference restarts in every clip that uses
|
||||
it. Use music only in the clip it belongs to, and when music must stop, say when in the
|
||||
summary, the shot, and the soundscape ("the music stops dead at the crash; silence after").
|
||||
9. **Stale text keeps steering.** After changing a shot, re-read the summary, retention lists, and
|
||||
soundscape for anything that now contradicts it ("slapping him on each beat") and fix it.
|
||||
10. **Do not add direction the script does not ask for.** No added shouting, roaring, or
|
||||
emphasis unless the script says so; all-caps dialogue may be written in normal case only if
|
||||
the user asks. Keep the script's dialogue verbatim except for obvious typos you report.
|
||||
|
||||
## Finding missing references
|
||||
|
||||
Only when `references.md` says you may use `find_reference_image`:
|
||||
|
||||
- First decide which subjects need a reference: every named person on screen, the main location,
|
||||
and any object whose exact look matters. A subject already covered by a listed file keeps that
|
||||
file; never search for it or replace it.
|
||||
- Search for the real thing the script names: a real person or actor by full name ("Robert
|
||||
Pattinson"), a place by what it is ("jazz rehearsal room interior"). For a character an actor
|
||||
plays, search the actor only when the user's direction names the actor.
|
||||
- Call it once per subject with `count` 2-3, look at every result with `read_image`, and use
|
||||
exactly one image per subject: a clear photograph of only that subject (see **Looking at
|
||||
references** for what to reject). Add "photo" to a place's query ("diner interior photo").
|
||||
- If nothing usable comes back, try different words once, then write the clip without a reference
|
||||
for that subject and say so in your summary.
|
||||
- In your final summary, list each downloaded file you used and the subject it is for.
|
||||
|
||||
## Looking at references
|
||||
|
||||
Use `read_image` on every image reference, listed or downloaded, before you write a
|
||||
`subject_definitions` line for it. The model copies what is in the picture, not what the script
|
||||
says, so write from what you see:
|
||||
|
||||
- A person: apparent age, build, hair, and the clothing and accessories they wear in the picture.
|
||||
If that clothing does not suit the scene (a red-carpet gown for a diner), keep the face and
|
||||
build and dress them as the script needs: say in the definition that the picture supplies the
|
||||
face only, and mark them `partially_preserved` in `retention_analysis`.
|
||||
- A place: what is in it, the layout, the light and its color, and whether people are in it. Say
|
||||
that it supplies the setting only, and whether the scene's extras fill it or it stays empty.
|
||||
- Anything in the picture the clip must not copy: its background, its lighting, text, a frame.
|
||||
|
||||
Reject a downloaded image that is a painting, drawing, poster or screenshot when the scene is live
|
||||
action; that shows several people when you need one; that shows the wrong person or place; or
|
||||
that is dominated by text or a watermark. Never reject a listed file, but tell the person in your
|
||||
summary when it does not match the script (wrong era, wrong clothing, more than one person).
|
||||
|
||||
If `read_image` is refused because the model cannot take images, carry on without it: describe a
|
||||
downloaded subject only from its file title and the script, invent no details such as clothing,
|
||||
and say in your summary that the references were not checked.
|
||||
|
||||
## Splitting into clips
|
||||
|
||||
- 5–14.4 s per clip. Cut at natural beats: a change of action, a new exchange, a cut in the script.
|
||||
- The first clip of a scene usually establishes the setting and characters (e.g. the band playing).
|
||||
- Put each major physical action (a throw, a slap sequence) where the clip can end right after it.
|
||||
- Characters keep the same position across clips unless the script moves them, and a move happens
|
||||
in the clip where the script puts it, not earlier.
|
||||
- Fixed camera ("The camera is fixed and does not move.") in every shot unless the script asks
|
||||
for movement.
|
||||
@@ -0,0 +1,346 @@
|
||||
<!-- MiniMax-H3 full-reference prompt guide, copied unchanged from
|
||||
https://huggingface.co/MiniMaxAI/MiniMax-H3/blob/42ed227ee7df40d41602854ae760620d6eb651fe/docs/VIDEO_PROMPT_WRITING_GUIDE_ref_en.md
|
||||
(via the dreamverse-harness video-harness branch). Where it disagrees with SKILL.md, SKILL.md wins:
|
||||
SKILL.md describes how FastVideo's pipeline actually labels references. -->
|
||||
|
||||
# Full-Reference Mode Rewrite Output Format Guide
|
||||
|
||||
This guide explains how rewrite outputs are organized and written in full-reference mode.
|
||||
|
||||
Write all six rewrite sections in English. Preserve the original language only for dialogue and lyrics inside `<d>` and for text visibly present in the scene.
|
||||
|
||||
**Description detail:** Make `detailed_description` as detailed and explicit as possible. For each shot, clearly establish the current composition, subject appearance and position, environment and lighting, actions and state changes, camera movement, current sound, and the points where referenced content actually appears or takes effect. Avoid reducing the description to a plot summary or a list of reference relationships.
|
||||
|
||||
> The basic formats for shots, camera movement, speakers, dialogue, and ordinary sound are shared with the Video Prompt Writing Guide (T2VA / I2VA / FL2VA / L2VA). This guide focuses on the reference labels, analysis sections, and format differences specific to full-reference mode.
|
||||
|
||||
## 1. Overall Structure
|
||||
|
||||
A complete rewrite output consists of six sections in the following order:
|
||||
|
||||
| Section | Purpose |
|
||||
| --- | --- |
|
||||
| `subject_definitions` | Defines referenced content and its reference labels |
|
||||
| `summary` | Summarizes the task type, target video, and main reference relationships |
|
||||
| `retention_analysis` | Describes how referenced content is preserved, transferred, or reused |
|
||||
| `detailed_description` | Describes visuals, actions, shots, sound, and dialogue in playback order |
|
||||
| `overall_soundscape` | Summarizes ambience and physical sounds |
|
||||
| `non_diegetic_music` | Describes background music audible only to the audience |
|
||||
|
||||
## 2. Reference Labels and Definitions (`subject_definitions`)
|
||||
|
||||
Full-reference rewrites use four types of labels to identify the source and role of referenced content:
|
||||
|
||||
| Label | Meaning |
|
||||
| --- | --- |
|
||||
| `<Subject N>` | Visible content abstracted from reference assets that can be reused or modified in the target video |
|
||||
| `<Picture N>` | A reference image used as a concrete target frame or shot-planning anchor |
|
||||
| `<Video N>` | A reference video that provides an editing source, continuation starting point, or whole-video temporal structure |
|
||||
| `<Audio N>` | An audio signal that is copied or referenced |
|
||||
|
||||
> Once a reference label is assigned to a piece of content, it keeps the same meaning across `subject_definitions`, `summary`, `retention_analysis`, `detailed_description`, and the audio sections.
|
||||
|
||||
`subject_definitions` defines each piece of referenced content that must be tracked separately later, such as a person, an environment, a source video's structure, or an audio track. Give each item its own line and explain what its label denotes, its reference role, and the main features to follow; name the corresponding source asset when its provenance needs to be made explicit. If `<Picture N>` or `<Video N>` only identifies the source of another referenced item and will not be analyzed or used separately later, cite it inside that item's definition without adding a separate line. `retention_analysis` records where each referenced item appears and whether it is fully preserved, partially preserved, transferred, or reused.
|
||||
|
||||
### 2.1 `<Subject N>`
|
||||
|
||||
`<Subject N>` is used for reusable visible content, including:
|
||||
|
||||
- People, animals, or objects
|
||||
- Scenes, backgrounds, or environments
|
||||
- Clothing, props, interfaces, or visual effects
|
||||
- Styles, actions, expressions, or poses
|
||||
|
||||
It represents a content unit that will actually be used in the target video, rather than the source file itself. One subject may be defined by multiple reference assets, and one reference asset may provide multiple subjects.
|
||||
|
||||
```text
|
||||
<Subject 1> is the young woman in <Picture 1>, with long dark hair, a blue cardigan, and a thin silver necklace.
|
||||
```
|
||||
|
||||
When the same subject comes from multiple assets, combine the sources and state what each asset provides:
|
||||
|
||||
```text
|
||||
<Subject 1> is the woman whose appearance comes from <Picture 1> and whose walking motion comes from <Video 1>.
|
||||
```
|
||||
|
||||
### 2.2 `<Picture N>`
|
||||
|
||||
Use a standalone `<Picture N>` when the reference image itself serves as a shot's first frame, keyframe, last frame, edited keyframe, or composition anchor:
|
||||
|
||||
```text
|
||||
<Picture 2> is the first frame of [Shot 1], showing a woman seated beside a café window.
|
||||
```
|
||||
|
||||
If an image is used only to define a character, scene, costume, or style, do not create a standalone picture entry. Instead, cite the image source inside the corresponding `<Subject N>` definition.
|
||||
|
||||
When an image acts as a storyboard or shot-planning reference, state which shots it maps to and what planning information it provides:
|
||||
|
||||
```text
|
||||
<Picture 3> is a storyboard reference for [Shot 1] and [Shot 2], defining their viewpoint, subject placement, and shot order.
|
||||
```
|
||||
|
||||
### 2.3 `<Video N>`
|
||||
|
||||
`<Video N>` is reserved for whole-video relationships, such as:
|
||||
|
||||
- Editing an original video
|
||||
- Continuing from the end of an original video
|
||||
- Referencing the original video's camera movement, cuts, rhythm, or temporal structure
|
||||
|
||||
```text
|
||||
<Video 1> is the source video for the target video edit.
|
||||
```
|
||||
|
||||
If a person, object, scene, action, or effect from a reference video is reused as visible content, it still belongs under `<Subject N>`. `<Video N>` identifies the asset or structural source and does not replace subject labels.
|
||||
|
||||
### 2.4 `<Audio N>`
|
||||
|
||||
`<Audio N>` represents a standalone audio asset or an enabled synchronized audio track from a reference video. Common uses include:
|
||||
|
||||
- Copying all or part of an audio signal
|
||||
- Referencing a background-music style
|
||||
- Referencing a speaker's voice timbre and delivery
|
||||
- Using dialogue, lyrics, or sound effects from the original audio
|
||||
- Referencing beat, rhythm, or audio continuity
|
||||
|
||||
When an `<Audio N>` explicitly corresponds to a target speaker, reuse that speaker's global ID in the definition: write `<Subject N> (Sx)` when the speaker maps to a defined subject, or use a stable voice description followed by `(Sx)` otherwise. The ID comes from the target video's global speaker order and is not independently assigned or renumbered in the audio definition. See Section 5.4 for the speaker-numbering rules:
|
||||
|
||||
```text
|
||||
<Audio 1> is the voice-timbre reference for <Subject 1> (S1).
|
||||
```
|
||||
|
||||
When one audio asset serves multiple roles, describe those roles in one natural sentence rather than creating additional subsections.
|
||||
|
||||
### 2.5 Visual and Audio Tracks from the Same Reference Video
|
||||
|
||||
`<Video N>` and `<Audio N>` are numbered independently. Each index indicates only the label's order within its own category and does not encode a pairing between the two categories. The same reference video may therefore correspond to `<Video 1>` and `<Audio 2>`; different indices do not prevent them from coming from the same source asset.
|
||||
|
||||
An ordinary reference video does not create `<Audio N>` merely because the file contains sound.
|
||||
|
||||
An `<Audio N>` definition primarily states the audio's role and does not have to name the `<Video N>` it comes from. State the shared source only when needed to remove provenance ambiguity, for example:
|
||||
|
||||
```text
|
||||
<Video 1> is the source video for the target video edit.
|
||||
<Audio 2> is the synchronized audio track of <Video 1> and is reused in the target video.
|
||||
```
|
||||
|
||||
## 3. `summary`
|
||||
|
||||
This section uses one short English paragraph to summarize the target video and its reference relationships. It begins with a square-bracketed task-type prefix:
|
||||
|
||||
```text
|
||||
[reference generation] ...
|
||||
[video editing + reference generation + audio reuse] ...
|
||||
```
|
||||
|
||||
Choose task types according to the actual role each reference asset plays in the target video:
|
||||
|
||||
| Task type | When to use it |
|
||||
| --- | --- |
|
||||
| `keyframe completion` | An image serves as the target video's first frame, keyframe, last frame, edited keyframe, or another concrete frame anchor |
|
||||
| `reference generation` | An image, video, or audio asset provides generation guidance for a character, scene, style, action, camera movement, storyboard, and so on, without serving as a concrete frame or as the source video being edited or continued |
|
||||
| `video editing` | An existing source video is directly modified; editing an image or generating between still keyframes does not belong to this type |
|
||||
| `video continuation` | New content continues, extends, resumes, or transitions from an existing source video |
|
||||
| `audio reuse` | The same audio signal is reused in full or in part |
|
||||
| `audio reference` | The audio signal is not copied directly; only its music style, timbre, dialogue or lyric content, sound-effect texture, beat, or continuity is referenced |
|
||||
|
||||
When a task satisfies multiple relationships, combine the task types with ` + ` and do not repeat a type. For example, continuing from a source video while using an image as the last frame is written as `[video continuation + keyframe completion]`. Editing a source video while retaining its original audio may be written as `[video editing + audio reuse]`.
|
||||
|
||||
The mere presence of video or audio does not automatically create a corresponding task type. If a reference video provides only camera movement, cuts, or rhythm, it normally belongs to `reference generation`. Use `video editing` or `video continuation` only when that video is directly edited or continued.
|
||||
|
||||
When editing a source video, use `audio reuse` as well if its original audio remains audible. When continuing a source video without directly copying the audio signal, use `audio reference` if the new audio only continues the original track's audible characteristics.
|
||||
|
||||
The summary uses the previously defined `<Subject N>`, `<Picture N>`, `<Video N>`, and `<Audio N>` labels to describe the main subjects, shot flow, and roles of the reference assets. Do not introduce new reference labels in this section.
|
||||
|
||||
For video-editing tasks, begin the summary after the task-type prefix with:
|
||||
|
||||
```text
|
||||
The target video is an edited version of <Video 1>.
|
||||
```
|
||||
|
||||
## 4. `retention_analysis`
|
||||
|
||||
This section describes how each piece of referenced content is preserved, transferred, copied, or referenced in the target video. Use one line for each reference label and preserve the meaning established in `subject_definitions`.
|
||||
|
||||
### 4.1 Visible Content
|
||||
|
||||
`<Subject N>`, `<Picture N>`, and `<Video N>` use the following relationship markers. These markers are fixed English values in the output format:
|
||||
|
||||
| Relationship marker | Meaning |
|
||||
| --- | --- |
|
||||
| `fully_preserved` | The defined role of the referenced content is fully preserved |
|
||||
| `partially_preserved` | The referenced content is still used, but some defined characteristics are changed or only partially retained |
|
||||
| `attribute_transfer` | Referenced characteristics are transferred to a different identifiable target subject |
|
||||
| `weak_reference` | Only broad similarity in style, category, composition, or atmosphere is retained |
|
||||
|
||||
Subject entry:
|
||||
|
||||
```text
|
||||
<Subject 1> (appears in [Shot 1], [Shot 3]): fully_preserved - ...
|
||||
```
|
||||
|
||||
Picture entry:
|
||||
|
||||
```text
|
||||
<Picture 2> ([Shot 1] first frame): fully_preserved - ...
|
||||
```
|
||||
|
||||
Video-structure entry:
|
||||
|
||||
```text
|
||||
<Video 1> (cut and pacing structure): weak_reference - ...
|
||||
```
|
||||
|
||||
### 4.2 Audio
|
||||
|
||||
`<Audio N>` uses the following relationship markers:
|
||||
|
||||
| Relationship marker | Meaning |
|
||||
| --- | --- |
|
||||
| `fully_copy` | The complete source audio serves as the target video's complete final audio track |
|
||||
| `partially_copy` | Only part of the timeline or selected audio layers are copied, or other sounds are added, removed, or replaced after copying |
|
||||
| `reference` | The signal is not copied directly; only timbre, rhythm, music style, dialogue content, or sound texture is referenced |
|
||||
| `weak_reference` | Only broad similarity in category or atmosphere is retained |
|
||||
|
||||
```text
|
||||
<Audio 1>: fully_copy - <Audio 1> is reused 1:1 as the target video's complete final audio track.
|
||||
```
|
||||
|
||||
```text
|
||||
<Audio 2>: reference - the target speaker follows <Audio 2>'s voice timbre and measured delivery without copying the original signal.
|
||||
```
|
||||
|
||||
Choose each relationship marker only within the reference role already defined for that label in `subject_definitions`. Do not treat newly added actions, backgrounds, or plot events in the target video as losses of reference fidelity.
|
||||
|
||||
## 5. `detailed_description`
|
||||
|
||||
This is the main body of a full-reference rewrite. It describes visuals, actions, sound, and dialogue shot by shot in target-video playback order and inserts reference labels where they apply.
|
||||
|
||||
### 5.1 Basic Format
|
||||
|
||||
The basic format follows the Video Prompt Writing Guide (T2VA / I2VA / FL2VA / L2VA):
|
||||
|
||||
- Write the body in English. Preserve the original language of dialogue, lyrics, and visible text.
|
||||
- `[Shot 1]` marks the opening shot and has no timestamp. Later shots use `[Shot N] At MM:SS.mmm, ...` to mark cut times.
|
||||
- Write camera movement as natural English within the current shot, including movement type, amplitude, and speed when they need to be expressed.
|
||||
- Give vocal sources stable `(S1)`, `(S2)`, and subsequent IDs. Write dialogue and lyrics as `<d>[Language] ...</d>`.
|
||||
- Use `<scenetrans>`, `<cutoff>`, and the corresponding continuity descriptions for dialogue crossing a cut, speech truncated by the video ending, and continuous audio across shots.
|
||||
|
||||
For complete rules and examples covering camera vocabulary, group speech, voice-over, dialogue across cuts, and visible text, see the Video Prompt Writing Guide (T2VA / I2VA / FL2VA / L2VA).
|
||||
|
||||
### 5.2 Full-Reference Mode Differences
|
||||
|
||||
| Dimension | T2VA | Full-reference mode |
|
||||
| --- | --- | --- |
|
||||
| Main field | `integrated_multimodal_description` | `detailed_description` |
|
||||
| Style opening | Written after `[Shot 1]` | Established in one or two English sentences before `[Shot 1]` |
|
||||
| Reference information | Does not use full-reference labels | Inserts `<Subject N>`, `<Picture N>`, `<Video N>`, and `<Audio N>` at their first appearance and where their roles apply |
|
||||
| Audio relationships | Describes the target video's own sound | Cites `<Audio N>` in the corresponding shot or audio phase and states whether the signal is copied or referenced |
|
||||
|
||||
Opening example:
|
||||
|
||||
```text
|
||||
The target video is in a cinematic, literary music-video style with soft lighting and a slightly desaturated color palette.
|
||||
[Shot 1] The scene opens in a crowded urban street...
|
||||
[Shot 2] At 00:09.000, the shot cuts to an extreme close-up...
|
||||
```
|
||||
|
||||
For generation tasks, `detailed_description` is normally 350-500 English words. Dialogue-dense content prioritizes fitting the complete spoken timeline rather than mechanically reaching a word count. Video-editing descriptions scale with the complexity of the source video and do not have to follow the generation-task range. A single shot does not automatically justify a shorter description; distribute detail across multiple shots according to their information load.
|
||||
|
||||
### 5.3 Using Reference Labels in Shots
|
||||
|
||||
At the first clear appearance of an important `<Subject N>`, describe its referenced characteristics, position in the frame, and current action within what is actually visible in the shot. Continue using the same label in later shots without redefining what the label represents.
|
||||
|
||||
Use natural phrasing for concrete frame anchors:
|
||||
|
||||
```text
|
||||
the shot begins from <Picture 1>
|
||||
the shot's keyframe corresponds to <Picture 2>
|
||||
the shot ends on <Picture 3>
|
||||
```
|
||||
|
||||
When editing or continuing an original video, cite `<Video N>` naturally where its source state, structure, or continuation relationship applies. Cite `<Audio N>` in the shot or semantic phase where the audio relationship is active.
|
||||
|
||||
### 5.4 Speakers, Audio Sources, and Dialogue
|
||||
|
||||
The basic speaker-ID and `<d>` formats follow T2VA. When a referenced subject physically speaks, retain both the visual reference label and the speaker ID:
|
||||
|
||||
```text
|
||||
<Subject 2> (S1) turns toward the woman and says, <d>[English] Last summer, I went to my grandfather's house. He talked about you.</d>
|
||||
```
|
||||
|
||||
`<Subject N>` identifies the referenced subject, while `(Sx)` identifies the actual speaker. When the subject speaks, write `<Subject N> (Sx)`. If the same subject speaks off-screen, keep the same form and mark it as `off-screen`. When the speaker does not correspond to a defined subject, use a stable voice description followed by `(Sx)`.
|
||||
|
||||
When verbal content is only a cue within a directly reused BGM or complete soundtrack, and no person, character, narrator, or other independent vocal source physically produces it, use `<Audio N>` as the audible source and do not invent an additional `(Sx)`. If a concrete person, character, narrator, or other independent vocal source produces the voice, assign and reuse `(Sx)` for that source:
|
||||
|
||||
```text
|
||||
When <Audio 1> reaches the phrase <d>[English] I'm lonely lonely lonely lonely lonely I'm lonely</d>, <Subject 1> performs the corresponding hand gesture without becoming a separate speaker source.
|
||||
```
|
||||
|
||||
When dialogue, narration, or lyrics from reference audio are directly reused, or when the input prompt explicitly requests their reperformance, preserve the exact source words and original language inside `<d>`. Write `[unclear]` for unintelligible spans instead of guessing or paraphrasing them. Standardize punctuation to the basic written marks needed to express the sentence, such as `,`, `.`, `?`, and `!`; remove repeated tildes, emoji, bullets, and repeated or decorative punctuation. End complete statements, questions, and exclamations with `.`, `?`, or `!` respectively before `</d>`.
|
||||
|
||||
When only timbre, rhythm, emotion, or delivery is referenced, do not carry the original dialogue from the reference audio into the target video.
|
||||
|
||||
Assign `(Sx)` once according to the order of actual vocal events in the target video. Reuse the corresponding ID at every actual vocal event in `detailed_description`; an `<Audio N>` definition bound to a target speaker in `subject_definitions` also reuses the same `(Sx)` but never assigns a new one independently. Do not write `(Sx)` in `retention_analysis`. Verbal cues that exist only within a directly reused BGM or complete soundtrack use `<Audio N>`; voices physically produced by a concrete person, character, narrator, or other independent vocal source use `(Sx)`.
|
||||
|
||||
## 6. `overall_soundscape` and `non_diegetic_music`
|
||||
|
||||
The definitions of these two sound categories follow the Video Prompt Writing Guide (T2VA / I2VA / FL2VA / L2VA).
|
||||
|
||||
`overall_soundscape` summarizes ambience and physical sounds across the full video. Dialogue, singing, and sound events synchronized to a particular shot remain in `detailed_description`:
|
||||
|
||||
```text
|
||||
overall_soundscape: Quiet indoor room tone and a low ventilation hum continue throughout the video.
|
||||
```
|
||||
|
||||
`non_diegetic_music` describes background music that the characters cannot hear and that is audible only to the audience. When music is present, state its instrumentation, tempo, and dynamic development:
|
||||
|
||||
```text
|
||||
non_diegetic_music: A restrained solo-piano score at a slow tempo, with sustained low cello underneath and no swell.
|
||||
```
|
||||
|
||||
When reference audio is used, state its copy or reference relationship only in the section that matches the audible layer: ambience and sound effects belong in `overall_soundscape`, while audience-only score belongs in `non_diegetic_music`. If the same audio provides both kinds of content, describe the corresponding relationship in each section:
|
||||
|
||||
```text
|
||||
overall_soundscape: The copied ambience layer from <Audio 1> continues throughout the target video.
|
||||
non_diegetic_music: <Audio 2> is directly reused as the complete audience-only score.
|
||||
```
|
||||
|
||||
Write complete dialogue and lyrics only inside `<d>` in `detailed_description`; do not repeat them in these two sections.
|
||||
|
||||
## 7. Complete Example
|
||||
|
||||
<details>
|
||||
<summary>Show the complete example</summary>
|
||||
|
||||
```text
|
||||
subject_definitions:
|
||||
<Subject 1> is the coffee-shop environment in <Picture 1>, featuring an exposed brick wall, an orange tufted sofa with patterned pillows, a neon sign, and a wooden coffee table.
|
||||
<Subject 2> is the fluffy white Samoyed in <Picture 2>, <Picture 3>, and <Picture 4>, with thick white fur, pointed ears, a dark nose, and a curved tail.
|
||||
<Subject 3> is the young blonde woman in <Video 1>, with long blonde hair and a light-pink button-down shirt with rolled-up sleeves.
|
||||
<Subject 4> is the young man in <Video 2>, with short wavy brown hair and a dark-grey hoodie with drawstrings.
|
||||
<Audio 1> is the voice-timbre reference for <Subject 3> (S1), containing a spoken English vocal layer.
|
||||
|
||||
summary:
|
||||
[reference generation + audio reference] The target video shows <Subject 3> eating a cookie in <Subject 1>. <Subject 4> enters with <Subject 2>, which lunges toward the cookie. The three-shot exchange uses <Audio 1> as the voice-timbre reference for <Subject 3> and ends with a canned audience laugh.
|
||||
|
||||
retention_analysis:
|
||||
<Subject 1> (appears in [Shot 1], [Shot 2], [Shot 3]): fully_preserved - the exposed brick wall, orange tufted sofa, patterned pillows, neon sign, and wooden coffee table are retained.
|
||||
<Subject 2> (appears in [Shot 1], [Shot 2]): fully_preserved - the Samoyed's thick white fur, pointed ears, dark nose, and curved tail are retained.
|
||||
<Subject 3> (appears in [Shot 1], [Shot 2], [Shot 3]): fully_preserved - the blonde woman's identity, long hair, and light-pink shirt are retained.
|
||||
<Subject 4> (appears in [Shot 1], [Shot 2]): fully_preserved - the young man's short wavy brown hair and dark-grey hoodie are retained.
|
||||
<Audio 1>: reference - its vocal timbre guides the dialogue delivery of <Subject 3> without copying the original signal.
|
||||
|
||||
detailed_description:
|
||||
The target video uses a realistic multi-camera sitcom style with warm indoor lighting.
|
||||
[Shot 1] A medium shot establishes <Subject 1>, the coffee shop with its exposed brick wall, orange tufted sofa, patterned pillows, neon sign, and wooden coffee table. <Subject 3> (S1), the young woman with long blonde hair and a light-pink button-down shirt with rolled-up sleeves, sits on the sofa holding a chocolate-chip cookie. From the left, <Subject 4>, the young man with short wavy brown hair and a dark-grey hoodie with drawstrings, enters holding the leash of <Subject 2>, the thick-furred white Samoyed with pointed ears, a dark nose, and a curved tail. The dog lunges toward the cookie and pulls the leash taut. <Subject 3> (S1) jerks her hand back and, using the clear youthful voice timbre referenced from <Audio 1>, exclaims with light annoyance, <d>[English] Hey! Watch your dog!</d> She closes her lips and guards the cookie while <Subject 4> pulls the dog back.
|
||||
[Shot 2] At 00:03.000, the shot cuts to a close-up of <Subject 4> (S2), the young man in the dark-grey hoodie from Shot 1, sitting beside <Subject 3> on the sofa and holding <Subject 2> securely in his arms. <Subject 4> (S2) says in a casual young male voice with a playful tone and an easy conversational pace, <d>[English] He just likes cookies more than me.</d> He closes his mouth into an apologetic smile and strokes the dog's thick white fur.
|
||||
[Shot 3] At 00:05.000, the shot cuts to a close-up of <Subject 3> (S1), the blonde woman in the light-pink shirt from Shot 1. Her annoyance softens as she looks toward the Samoyed. <Subject 3> (S1) replies in the same clear youthful voice referenced from <Audio 1> with an amused cadence, <d>[English] Well, he has good taste at least.</d> She smiles and raises the cookie in a small toast-like gesture. A classic canned audience laugh begins immediately after the line and continues through the final frame.
|
||||
|
||||
overall_soundscape:
|
||||
Soft indoor coffee-shop room tone continues throughout the scene.
|
||||
|
||||
non_diegetic_music:
|
||||
N/A
|
||||
```
|
||||
|
||||
</details>
|
||||
@@ -37,6 +37,8 @@ from fastvideo_studio.frames import FrameError, last_clip_for_job, last_frame_fo
|
||||
from fastvideo_studio.gpu import get_gpu_snapshot
|
||||
from fastvideo_studio.job_runner import JobRunner, JobStatus
|
||||
from fastvideo_studio.merge import MergeError, merge_jobs, merged_path
|
||||
from fastvideo_studio.scene_writer.routes import router as scene_writer_router
|
||||
from fastvideo_studio.scene_writer.runner import use_server_data_dir
|
||||
from fastvideo_studio.models import (CreateDatasetRequest, CreateJobRequest, MergeSceneRequest, QueueJobsRequest,
|
||||
SettingsUpdate, TrimRequest, UpdateCaptionRequest, model_label)
|
||||
|
||||
@@ -72,6 +74,8 @@ app.add_middleware(
|
||||
allow_headers=["*"],
|
||||
)
|
||||
|
||||
app.include_router(scene_writer_router)
|
||||
|
||||
|
||||
@app.get("/api/settings")
|
||||
def get_settings() -> dict[str, Any]:
|
||||
@@ -914,6 +918,7 @@ def main() -> None:
|
||||
data_dir = Path(args.data_dir).resolve()
|
||||
upload_dir = str(data_dir / "uploads")
|
||||
datasets_upload_dir = str(data_dir / "uploads" / "datasets")
|
||||
use_server_data_dir(data_dir)
|
||||
|
||||
create_local_env(args.host, args.port)
|
||||
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
'use client';
|
||||
|
||||
import { AlertTriangle, Loader2, RefreshCw } from 'lucide-react';
|
||||
import { AlertTriangle, Loader2, PenLine, RefreshCw } from 'lucide-react';
|
||||
import * as React from 'react';
|
||||
|
||||
import CreateJobModal from '@/components/jobs/CreateJobModal';
|
||||
import { EDITABLE_STATUSES, SceneBoard } from '@/components/scenes/SceneBoard';
|
||||
import { MergedSceneDialog } from '@/components/scenes/MergedSceneDialog';
|
||||
import { SceneQueueDialog } from '@/components/scenes/SceneQueueDialog';
|
||||
import { ReviseClipDialog } from '@/components/scenes/ReviseClipDialog';
|
||||
import { TrimClipDialog } from '@/components/scenes/TrimClipDialog';
|
||||
import { WriteSceneDialog } from '@/components/scenes/WriteSceneDialog';
|
||||
import { Button } from '@/components/ui/button';
|
||||
import { Card } from '@/components/ui/card';
|
||||
import { NativeSelect } from '@/components/ui/native-select';
|
||||
@@ -29,9 +31,11 @@ export default function ScenesPage() {
|
||||
const [openJob, setOpenJob] = React.useState<Job | null>(null);
|
||||
// which re-run of a clip number the user picked, by clip
|
||||
const [choices, setChoices] = React.useState<Record<string, string>>({});
|
||||
const [writing, setWriting] = React.useState(false);
|
||||
const [queueTarget, setQueueTarget] = React.useState<Scene | null>(null);
|
||||
const [merged, setMerged] = React.useState<(MergedScene & { title: string }) | null>(null);
|
||||
const [trimTarget, setTrimTarget] = React.useState<Job | null>(null);
|
||||
const [reviseTarget, setReviseTarget] = React.useState<SceneClip | null>(null);
|
||||
// Bumped per job id whenever its video is edited in place, to cache-bust every
|
||||
// <video> for that job (the dialog's own preview and the scene board's). Owned
|
||||
// here, not by the dialog, so it survives the dialog re-rendering with a fresh
|
||||
@@ -172,6 +176,15 @@ export default function ScenesPage() {
|
||||
|
||||
const scenes = React.useMemo(() => buildScenes(jobs, choices), [jobs, choices]);
|
||||
const scene = scenes.find((s) => s.key === selectedKey) ?? scenes[0];
|
||||
// The shown takes of the clips on either side of the one being revised.
|
||||
const reviseNeighbors = React.useMemo(() => {
|
||||
if (!reviseTarget) return [];
|
||||
for (const s of scenes) {
|
||||
const i = s.clips.findIndex((c) => c.job.id === reviseTarget.job.id);
|
||||
if (i >= 0) return [s.clips[i - 1], s.clips[i + 1]].filter((c) => c !== undefined).map((c) => c.job);
|
||||
}
|
||||
return [];
|
||||
}, [scenes, reviseTarget]);
|
||||
|
||||
return (
|
||||
<div className="mx-auto w-full max-w-[1200px] px-4 pb-12 pt-4">
|
||||
@@ -183,6 +196,16 @@ export default function ScenesPage() {
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="ml-auto gap-1.5"
|
||||
onClick={() => setWriting(true)}
|
||||
>
|
||||
<PenLine className="size-3.5" aria-hidden />
|
||||
Write scene
|
||||
</Button>
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="gap-1.5"
|
||||
onClick={reload}
|
||||
disabled={isLoading}
|
||||
>
|
||||
@@ -237,6 +260,7 @@ export default function ScenesPage() {
|
||||
onQueueClip={queueClip}
|
||||
onDequeueClip={dequeueClip}
|
||||
onTrimClip={(clip) => setTrimTarget(clip.job)}
|
||||
onReviseClip={setReviseTarget}
|
||||
onChooseTake={(takeKey, jobId) => setChoices((c) => ({ ...c, [takeKey]: jobId }))}
|
||||
videoVersionFor={(jobId) => videoVersions[jobId] ?? 0}
|
||||
/>
|
||||
@@ -245,6 +269,13 @@ export default function ScenesPage() {
|
||||
</Card>
|
||||
|
||||
<MergedSceneDialog merged={merged} onClose={() => setMerged(null)} />
|
||||
<WriteSceneDialog open={writing} onClose={() => setWriting(false)} onCreated={reload} />
|
||||
<ReviseClipDialog
|
||||
clip={reviseTarget}
|
||||
neighbors={reviseNeighbors}
|
||||
onClose={() => setReviseTarget(null)}
|
||||
onCreated={reload}
|
||||
/>
|
||||
<SceneQueueDialog scene={queueTarget} onConfirm={queueScene} onClose={() => setQueueTarget(null)} />
|
||||
<TrimClipDialog
|
||||
clip={trimTarget}
|
||||
|
||||
@@ -0,0 +1,230 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { Loader2 } from 'lucide-react';
|
||||
import { toast } from 'sonner';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import {
|
||||
duplicateJob,
|
||||
getClipRevision,
|
||||
startClipRevision,
|
||||
updateJob,
|
||||
type ClipRevision,
|
||||
type DraftedJob,
|
||||
} from '@/lib/api';
|
||||
import { formatDuration } from '@/lib/jobTiming';
|
||||
import { diffLines } from '@/lib/lineDiff';
|
||||
import type { SceneClip } from '@/lib/scenes';
|
||||
import type { Job } from '@/lib/types';
|
||||
|
||||
const POLL_MS = 3000;
|
||||
|
||||
function seconds(job: Pick<DraftedJob, 'num_frames' | 'fps'>): string {
|
||||
return `${((job.num_frames ?? 0) / (job.fps || 24)).toFixed(1)} s`;
|
||||
}
|
||||
|
||||
/** Changes outside the prompt text, as short sentences. */
|
||||
function settingChanges(before: DraftedJob, after: DraftedJob): string[] {
|
||||
const changes: string[] = [];
|
||||
if (before.num_frames !== after.num_frames) {
|
||||
changes.push(`Length ${seconds(before)} → ${seconds(after)} (${before.num_frames} → ${after.num_frames} frames)`);
|
||||
}
|
||||
const sources = (j: DraftedJob) => (j.references ?? []).map((r) => r.source);
|
||||
const was = sources(before);
|
||||
const now = sources(after);
|
||||
for (const s of now.filter((s) => !was.includes(s))) changes.push(`Added reference ${s.split('/').pop()}`);
|
||||
for (const s of was.filter((s) => !now.includes(s))) changes.push(`Removed reference ${s.split('/').pop()}`);
|
||||
if (was.length === now.length && was.some((s, i) => s !== now[i])) changes.push('Reordered the references');
|
||||
if (before.seed !== after.seed) changes.push(`Seed ${before.seed} → ${after.seed}`);
|
||||
return changes;
|
||||
}
|
||||
|
||||
export interface ReviseClipDialogProps {
|
||||
/** the clip to revise; its shown take is the one revised */
|
||||
clip: SceneClip | null;
|
||||
/** the shown takes of the clips before and after it, for continuity */
|
||||
neighbors: Job[];
|
||||
onClose: () => void;
|
||||
/** called after the revision is created as a new take */
|
||||
onCreated: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Describe what went wrong in a clip's take; the scene writer revises the
|
||||
* clip's job, and the revision becomes a new pending take of the same clip
|
||||
* (same name), leaving the old take as it is.
|
||||
*/
|
||||
export function ReviseClipDialog({ clip, neighbors, onClose, onCreated }: ReviseClipDialogProps) {
|
||||
const [feedback, setFeedback] = React.useState('');
|
||||
const [revision, setRevision] = React.useState<ClipRevision | null>(null);
|
||||
const [starting, setStarting] = React.useState(false);
|
||||
const [creating, setCreating] = React.useState(false);
|
||||
const [now, setNow] = React.useState(() => Date.now());
|
||||
|
||||
// Hiding the dialog keeps a running revision; opening it for a different take starts fresh.
|
||||
const jobId = clip?.job.id;
|
||||
const lastJobId = React.useRef<string | undefined>(undefined);
|
||||
React.useEffect(() => {
|
||||
if (jobId === undefined || jobId === lastJobId.current) return;
|
||||
lastJobId.current = jobId;
|
||||
setFeedback('');
|
||||
setRevision(null);
|
||||
}, [jobId]);
|
||||
|
||||
const running = revision?.status === 'running';
|
||||
React.useEffect(() => {
|
||||
if (!running || !revision) return;
|
||||
const runId = revision.run_id;
|
||||
const poll = setInterval(async () => {
|
||||
try {
|
||||
setRevision(await getClipRevision(runId));
|
||||
} catch {
|
||||
// Keep the last state; the next poll retries.
|
||||
}
|
||||
}, POLL_MS);
|
||||
const tick = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => {
|
||||
clearInterval(poll);
|
||||
clearInterval(tick);
|
||||
};
|
||||
}, [running, revision?.run_id]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
if (!clip) return null;
|
||||
const { job } = clip;
|
||||
|
||||
async function start() {
|
||||
setStarting(true);
|
||||
try {
|
||||
setRevision(await startClipRevision({ job, feedback, neighbors }));
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't start the revision.");
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function createTake() {
|
||||
if (!revision?.job) return;
|
||||
setCreating(true);
|
||||
try {
|
||||
const { id } = await duplicateJob(job.id);
|
||||
await updateJob(id, {
|
||||
prompt: revision.job.prompt.join('\n'),
|
||||
num_frames: revision.job.num_frames,
|
||||
references: revision.job.references ?? [],
|
||||
seed: revision.job.seed,
|
||||
});
|
||||
toast.success(`Created a new take of ${job.name}. It is pending; queue it when ready.`);
|
||||
onCreated();
|
||||
onClose();
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't create the take.");
|
||||
} finally {
|
||||
setCreating(false);
|
||||
}
|
||||
}
|
||||
|
||||
const done = revision?.status === 'completed' && revision.job !== null;
|
||||
const elapsed = revision ? (revision.finished_at ?? now / 1000) - revision.started_at : 0;
|
||||
const diff = done ? diffLines(revision.original.prompt.join('\n'), revision.job!.prompt.join('\n')) : [];
|
||||
const changes = done ? settingChanges(revision.original, revision.job!) : [];
|
||||
|
||||
return (
|
||||
<Dialog open onOpenChange={(o) => !o && !creating && onClose()}>
|
||||
<DialogContent className="max-h-[90vh] max-w-3xl overflow-y-auto">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Revise {job.name}</DialogTitle>
|
||||
<DialogDescription>
|
||||
Describe what went wrong in this take. The scene writer finds the cause in the prompt and rewrites it; you
|
||||
review the changes, then create them as a new take of the same clip. The current take is kept.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="flex flex-col gap-4 text-sm">
|
||||
<label className="flex flex-col gap-1">
|
||||
What went wrong
|
||||
<Textarea
|
||||
value={feedback}
|
||||
onChange={(e) => setFeedback(e.target.value)}
|
||||
rows={4}
|
||||
placeholder="e.g. There are 4 slaps instead of 2. / Andrew appears twice. / The band disappears after the chair throw."
|
||||
disabled={running}
|
||||
/>
|
||||
</label>
|
||||
|
||||
{revision && (
|
||||
<section aria-live="polite" className="flex flex-col gap-3 rounded-lg border border-border p-3">
|
||||
<div className="flex items-center gap-2">
|
||||
{running && <Loader2 className="size-4 animate-spin text-primary" aria-hidden />}
|
||||
<span className="font-medium">
|
||||
{running ? 'Revising…' : done ? 'Revision ready' : 'Revision failed'}
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">{formatDuration(elapsed)}</span>
|
||||
</div>
|
||||
{revision.error && <p className="text-xs text-destructive">{revision.error}</p>}
|
||||
{revision.final_response && (
|
||||
<p className="whitespace-pre-wrap text-xs">{revision.final_response}</p>
|
||||
)}
|
||||
{changes.length > 0 && (
|
||||
<ul className="list-disc pl-5 text-xs">
|
||||
{changes.map((c) => (
|
||||
<li key={c}>{c}</li>
|
||||
))}
|
||||
</ul>
|
||||
)}
|
||||
{done && (
|
||||
<pre className="max-h-96 overflow-auto whitespace-pre-wrap rounded bg-background p-2 text-[11px]">
|
||||
{diff.map((line, i) => (
|
||||
<div
|
||||
key={i}
|
||||
className={
|
||||
line.kind === 'added'
|
||||
? 'bg-green-500/15 text-green-700 dark:text-green-400'
|
||||
: line.kind === 'removed'
|
||||
? 'bg-red-500/15 text-red-700 line-through dark:text-red-400'
|
||||
: 'text-muted-foreground'
|
||||
}
|
||||
>
|
||||
{line.kind === 'added' ? '+ ' : line.kind === 'removed' ? '- ' : ' '}
|
||||
{line.text}
|
||||
</div>
|
||||
))}
|
||||
</pre>
|
||||
)}
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<DialogFooter>
|
||||
<Button type="button" variant="outline" onClick={onClose} disabled={creating}>
|
||||
{running ? 'Hide' : 'Cancel'}
|
||||
</Button>
|
||||
{done ? (
|
||||
<>
|
||||
<Button type="button" variant="outline" onClick={start} disabled={!feedback.trim() || creating}>
|
||||
Revise again
|
||||
</Button>
|
||||
<Button type="button" onClick={createTake} disabled={creating}>
|
||||
{creating ? 'Creating…' : 'Create new take'}
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<Button type="button" onClick={start} disabled={!feedback.trim() || running || starting}>
|
||||
{starting ? 'Starting…' : running ? 'Revising…' : 'Revise'}
|
||||
</Button>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -117,6 +117,7 @@ function ClipRow({
|
||||
onQueueClip,
|
||||
onDequeueClip,
|
||||
onTrimClip,
|
||||
onReviseClip,
|
||||
videoVersionFor,
|
||||
}: {
|
||||
clip: SceneClip;
|
||||
@@ -130,6 +131,7 @@ function ClipRow({
|
||||
onQueueClip: (clip: SceneClip) => Promise<void>;
|
||||
onDequeueClip: (clip: SceneClip) => Promise<void>;
|
||||
onTrimClip: (clip: SceneClip) => void;
|
||||
onReviseClip: (clip: SceneClip) => void;
|
||||
videoVersionFor: (jobId: string) => number;
|
||||
}) {
|
||||
const [previewing, setPreviewing] = React.useState(false);
|
||||
@@ -264,6 +266,17 @@ function ClipRow({
|
||||
Edit video
|
||||
</Button>
|
||||
)}
|
||||
{canPreview && (
|
||||
<Button
|
||||
type="button"
|
||||
size="sm"
|
||||
variant="outline"
|
||||
onClick={() => onReviseClip(clip)}
|
||||
title="Describe what went wrong; the scene writer rewrites the prompt as a new take"
|
||||
>
|
||||
Revise
|
||||
</Button>
|
||||
)}
|
||||
{canQueue && (
|
||||
<Button
|
||||
type="button"
|
||||
@@ -341,6 +354,8 @@ export interface SceneBoardProps {
|
||||
onDequeueClip: (clip: SceneClip) => Promise<void>;
|
||||
/** open the trim dialog for one clip's video */
|
||||
onTrimClip: (clip: SceneClip) => void;
|
||||
/** open the revise dialog for one clip's shown take */
|
||||
onReviseClip: (clip: SceneClip) => void;
|
||||
/** cache-busting version for a job's video preview, bumped after each edit */
|
||||
videoVersionFor: (jobId: string) => number;
|
||||
}
|
||||
@@ -357,6 +372,7 @@ export function SceneBoard({
|
||||
onQueueClip,
|
||||
onDequeueClip,
|
||||
onTrimClip,
|
||||
onReviseClip,
|
||||
videoVersionFor,
|
||||
}: SceneBoardProps) {
|
||||
let shotNumber = 1;
|
||||
@@ -457,6 +473,7 @@ export function SceneBoard({
|
||||
onQueueClip={onQueueClip}
|
||||
onDequeueClip={onDequeueClip}
|
||||
onTrimClip={onTrimClip}
|
||||
onReviseClip={onReviseClip}
|
||||
videoVersionFor={videoVersionFor}
|
||||
/>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,389 @@
|
||||
'use client';
|
||||
|
||||
import * as React from 'react';
|
||||
import { Loader2, Plus, Trash2, Upload } from 'lucide-react';
|
||||
import { toast } from 'sonner';
|
||||
|
||||
import { Button } from '@/components/ui/button';
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogFooter,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from '@/components/ui/dialog';
|
||||
import { Input } from '@/components/ui/input';
|
||||
import { NativeSelect } from '@/components/ui/native-select';
|
||||
import { Textarea } from '@/components/ui/textarea';
|
||||
import {
|
||||
createJob,
|
||||
getFetchedReferenceUrl,
|
||||
getSceneDraft,
|
||||
startSceneDraft,
|
||||
uploadMedia,
|
||||
type DraftedJob,
|
||||
type MediaType,
|
||||
type SceneDraft,
|
||||
type SceneDraftReference,
|
||||
} from '@/lib/api';
|
||||
import { formatDuration } from '@/lib/jobTiming';
|
||||
|
||||
const POLL_MS = 3000;
|
||||
const SCENE_NAME_RE = /^[a-z0-9]+(?:-[a-z0-9]+)*$/;
|
||||
|
||||
function emptyReference(): SceneDraftReference {
|
||||
return { path: '', media_type: 'image', description: '' };
|
||||
}
|
||||
|
||||
/** Clip length and shot count, read from the drafted job. */
|
||||
function describeJob(job: DraftedJob): string {
|
||||
const seconds = (job.num_frames ?? 0) / (job.fps || 24);
|
||||
const shots = job.prompt.filter((l) => /^\[Shot \d+\]/.test(l)).length;
|
||||
return `${seconds.toFixed(1)}s · ${shots} shot${shots === 1 ? '' : 's'}`;
|
||||
}
|
||||
|
||||
function summaryOf(job: DraftedJob): string {
|
||||
const i = job.prompt.indexOf('summary:');
|
||||
return (i >= 0 ? job.prompt[i + 1] : '')?.replace(/^\[[^\]]*\]\s*/, '') ?? '';
|
||||
}
|
||||
|
||||
export interface WriteSceneDialogProps {
|
||||
open: boolean;
|
||||
onClose: () => void;
|
||||
/** Called after drafts are created as pending jobs. */
|
||||
onCreated: () => void;
|
||||
}
|
||||
|
||||
/**
|
||||
* Draft a scene's jobs from a script with the scene writer agent, review the
|
||||
* drafts, then create them as pending jobs. Nothing is created or started
|
||||
* until "Create jobs".
|
||||
*/
|
||||
export function WriteSceneDialog({ open, onClose, onCreated }: WriteSceneDialogProps) {
|
||||
const [sceneName, setSceneName] = React.useState('');
|
||||
const [script, setScript] = React.useState('');
|
||||
const [references, setReferences] = React.useState<SceneDraftReference[]>([emptyReference()]);
|
||||
const [instructions, setInstructions] = React.useState('');
|
||||
const [findReferences, setFindReferences] = React.useState(false);
|
||||
const [draft, setDraft] = React.useState<SceneDraft | null>(null);
|
||||
const [starting, setStarting] = React.useState(false);
|
||||
const [creating, setCreating] = React.useState(false);
|
||||
const [expanded, setExpanded] = React.useState<string | null>(null);
|
||||
const [now, setNow] = React.useState(() => Date.now());
|
||||
|
||||
const running = draft?.status === 'running';
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!running || !draft) return;
|
||||
const runId = draft.run_id;
|
||||
const poll = setInterval(async () => {
|
||||
try {
|
||||
setDraft(await getSceneDraft(runId));
|
||||
} catch {
|
||||
// Keep the last state; the next poll retries.
|
||||
}
|
||||
}, POLL_MS);
|
||||
const tick = setInterval(() => setNow(Date.now()), 1000);
|
||||
return () => {
|
||||
clearInterval(poll);
|
||||
clearInterval(tick);
|
||||
};
|
||||
}, [running, draft?.run_id]); // eslint-disable-line react-hooks/exhaustive-deps
|
||||
|
||||
const usableRefs = references.filter((r) => r.path.trim());
|
||||
const nameValid = SCENE_NAME_RE.test(sceneName);
|
||||
const canStart =
|
||||
nameValid && script.trim().length > 0 && (usableRefs.length > 0 || findReferences) && !running && !starting;
|
||||
|
||||
function updateReference(index: number, patch: Partial<SceneDraftReference>) {
|
||||
setReferences((refs) => refs.map((r, i) => (i === index ? { ...r, ...patch } : r)));
|
||||
}
|
||||
|
||||
async function upload(index: number, file: File) {
|
||||
try {
|
||||
const { path, media_type } = await uploadMedia(file);
|
||||
updateReference(index, { path, media_type });
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : 'Upload failed');
|
||||
}
|
||||
}
|
||||
|
||||
async function start() {
|
||||
setStarting(true);
|
||||
try {
|
||||
setExpanded(null);
|
||||
setDraft(
|
||||
await startSceneDraft({
|
||||
script,
|
||||
scene_name: sceneName,
|
||||
references: usableRefs.map((r) => ({ ...r, path: r.path.trim() })),
|
||||
instructions,
|
||||
find_references: findReferences,
|
||||
}),
|
||||
);
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? e.message : "Couldn't start the draft.");
|
||||
} finally {
|
||||
setStarting(false);
|
||||
}
|
||||
}
|
||||
|
||||
async function createAll() {
|
||||
if (!draft) return;
|
||||
setCreating(true);
|
||||
try {
|
||||
for (const job of draft.jobs) {
|
||||
await createJob({ ...job, prompt: job.prompt.join('\n') });
|
||||
}
|
||||
const n = draft.jobs.length;
|
||||
toast.success(`Created ${n} pending job${n === 1 ? '' : 's'}. Review them, then queue the scene.`);
|
||||
onCreated();
|
||||
setDraft(null);
|
||||
onClose();
|
||||
} catch (e) {
|
||||
toast.error(e instanceof Error ? `${e.message}. Jobs created before the error remain.` : "Couldn't create the jobs.");
|
||||
} finally {
|
||||
setCreating(false);
|
||||
}
|
||||
}
|
||||
|
||||
const elapsed = draft ? ((draft.finished_at ?? now / 1000) - draft.started_at) : 0;
|
||||
|
||||
return (
|
||||
<Dialog open={open} onOpenChange={(o) => !o && !creating && onClose()}>
|
||||
<DialogContent className="max-h-[90vh] max-w-3xl overflow-y-auto">
|
||||
<DialogHeader>
|
||||
<DialogTitle>Write a scene from a script</DialogTitle>
|
||||
<DialogDescription>
|
||||
The scene writer splits the script into clips and drafts an H3 job for each. Drafts are only created as
|
||||
pending jobs when you press Create.
|
||||
</DialogDescription>
|
||||
</DialogHeader>
|
||||
|
||||
<div className="flex flex-col gap-4 text-sm">
|
||||
<label className="flex flex-col gap-1">
|
||||
Scene name
|
||||
<Input
|
||||
value={sceneName}
|
||||
onChange={(e) => setSceneName(e.target.value.toLowerCase())}
|
||||
placeholder="whiplash-rehearsal"
|
||||
disabled={running}
|
||||
aria-invalid={sceneName !== '' && !nameValid}
|
||||
/>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
Clips are named <code className="font-mono">{sceneName || 'scene'}-clip-01</code>,{' '}
|
||||
<code className="font-mono">-02</code>, … Lowercase letters, digits and dashes.
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<label className="flex flex-col gap-1">
|
||||
Script
|
||||
<Textarea
|
||||
value={script}
|
||||
onChange={(e) => setScript(e.target.value)}
|
||||
rows={10}
|
||||
className="font-mono text-xs"
|
||||
placeholder="Paste the scene: action lines, character names and dialogue."
|
||||
disabled={running}
|
||||
/>
|
||||
</label>
|
||||
|
||||
<fieldset className="flex flex-col gap-2" disabled={running}>
|
||||
<legend className="mb-1">References</legend>
|
||||
{references.map((ref, i) => (
|
||||
<div key={i} className="flex flex-wrap items-center gap-2">
|
||||
<Input
|
||||
value={ref.path}
|
||||
onChange={(e) => updateReference(i, { path: e.target.value })}
|
||||
placeholder="/absolute/path/to/reference.png"
|
||||
className="min-w-[16rem] flex-[2] font-mono text-xs"
|
||||
aria-label={`Reference ${i + 1} path`}
|
||||
/>
|
||||
<NativeSelect
|
||||
value={ref.media_type}
|
||||
onChange={(e) => updateReference(i, { media_type: e.target.value as MediaType })}
|
||||
aria-label={`Reference ${i + 1} type`}
|
||||
className="w-24"
|
||||
>
|
||||
<option value="image">image</option>
|
||||
<option value="audio">audio</option>
|
||||
<option value="video">video</option>
|
||||
</NativeSelect>
|
||||
<Input
|
||||
value={ref.description}
|
||||
onChange={(e) => updateReference(i, { description: e.target.value })}
|
||||
placeholder="What it is, e.g. Fletcher's face / Andrew's voice / the empty rehearsal room"
|
||||
className="min-w-[12rem] flex-[2]"
|
||||
aria-label={`Reference ${i + 1} description`}
|
||||
/>
|
||||
<label className="inline-flex cursor-pointer items-center" title="Upload a file">
|
||||
<Upload className="size-4 text-muted-foreground" aria-hidden />
|
||||
<span className="sr-only">Upload reference {i + 1}</span>
|
||||
<input
|
||||
type="file"
|
||||
accept="image/*,audio/*,video/*"
|
||||
className="sr-only"
|
||||
onChange={(e) => e.target.files?.[0] && upload(i, e.target.files[0])}
|
||||
/>
|
||||
</label>
|
||||
<Button
|
||||
type="button"
|
||||
variant="ghost"
|
||||
size="sm"
|
||||
onClick={() => setReferences((refs) => refs.filter((_, j) => j !== i))}
|
||||
aria-label={`Remove reference ${i + 1}`}
|
||||
disabled={references.length === 1}
|
||||
>
|
||||
<Trash2 className="size-4" aria-hidden />
|
||||
</Button>
|
||||
</div>
|
||||
))}
|
||||
<Button
|
||||
type="button"
|
||||
variant="outline"
|
||||
size="sm"
|
||||
className="w-fit gap-1.5"
|
||||
onClick={() => setReferences((refs) => [...refs, emptyReference()])}
|
||||
>
|
||||
<Plus className="size-3.5" aria-hidden />
|
||||
Add reference
|
||||
</Button>
|
||||
</fieldset>
|
||||
|
||||
<label className="flex items-start gap-2">
|
||||
<input
|
||||
type="checkbox"
|
||||
checked={findReferences}
|
||||
onChange={(e) => setFindReferences(e.target.checked)}
|
||||
disabled={running}
|
||||
className="mt-1 size-4"
|
||||
/>
|
||||
<span>
|
||||
Find missing references online
|
||||
<span className="block text-xs text-muted-foreground">
|
||||
The writer may download photos from Wikimedia Commons for people, places or objects in the script that
|
||||
none of your references show. You can review each one, with its source and license, before creating
|
||||
jobs.
|
||||
</span>
|
||||
</span>
|
||||
</label>
|
||||
|
||||
<label className="flex flex-col gap-1">
|
||||
Direction (optional)
|
||||
<Textarea
|
||||
value={instructions}
|
||||
onChange={(e) => setInstructions(e.target.value)}
|
||||
rows={3}
|
||||
placeholder="e.g. The first clip is just the band playing; the chair throw comes after."
|
||||
disabled={running}
|
||||
/>
|
||||
</label>
|
||||
|
||||
{draft && (
|
||||
<section aria-live="polite" className="flex flex-col gap-2 rounded-lg border border-border p-3">
|
||||
<div className="flex items-center gap-2">
|
||||
{running && <Loader2 className="size-4 animate-spin text-primary" aria-hidden />}
|
||||
<span className="font-medium">
|
||||
{draft.status === 'running'
|
||||
? 'Drafting…'
|
||||
: draft.status === 'completed'
|
||||
? 'Draft ready'
|
||||
: 'Draft failed'}
|
||||
</span>
|
||||
<span className="text-xs text-muted-foreground">
|
||||
{formatDuration(elapsed)} · {draft.jobs.length} clip{draft.jobs.length === 1 ? '' : 's'} written
|
||||
</span>
|
||||
</div>
|
||||
{draft.error && <p className="text-xs text-destructive">{draft.error}</p>}
|
||||
{draft.status === 'completed' && draft.jobs.length === 0 && (
|
||||
<p className="text-xs text-muted-foreground">
|
||||
The writer finished without writing any clips. {draft.final_response ?? ''}
|
||||
</p>
|
||||
)}
|
||||
{draft.fetched_references.length > 0 && (
|
||||
<div className="flex flex-col gap-1.5">
|
||||
<span className="text-xs font-medium">
|
||||
Downloaded references ({draft.fetched_references.length}): check these before creating jobs
|
||||
</span>
|
||||
<ul className="grid grid-cols-2 gap-2 sm:grid-cols-4">
|
||||
{draft.fetched_references.map((ref) => {
|
||||
const used = draft.jobs.some((j) => j.references?.some((r) => r.source === ref.path));
|
||||
return (
|
||||
<li key={ref.path} className="flex flex-col gap-1 rounded-md bg-secondary/40 p-1.5 text-[11px]">
|
||||
{/* eslint-disable-next-line @next/next/no-img-element -- served by the studio API, not Next */}
|
||||
<img
|
||||
src={getFetchedReferenceUrl(draft.run_id, ref)}
|
||||
alt={`${ref.subject}: ${ref.title}`}
|
||||
className="aspect-square w-full rounded object-cover"
|
||||
/>
|
||||
<span className="font-medium">
|
||||
{ref.subject}
|
||||
{used ? ' · used' : ''}
|
||||
</span>
|
||||
<a
|
||||
href={ref.page}
|
||||
target="_blank"
|
||||
rel="noreferrer"
|
||||
className="truncate text-muted-foreground underline"
|
||||
title={ref.title}
|
||||
>
|
||||
{ref.title}
|
||||
</a>
|
||||
<span className="truncate text-muted-foreground" title={`${ref.license} · ${ref.artist}`}>
|
||||
{ref.license} · {ref.artist}
|
||||
</span>
|
||||
</li>
|
||||
);
|
||||
})}
|
||||
</ul>
|
||||
</div>
|
||||
)}
|
||||
<ol className="flex flex-col gap-1.5">
|
||||
{draft.jobs.map((job) => (
|
||||
<li key={job.name} className="rounded-md bg-secondary/40 p-2">
|
||||
<button
|
||||
type="button"
|
||||
className="flex w-full flex-wrap items-baseline gap-2 text-left"
|
||||
onClick={() => setExpanded((n) => (n === job.name ? null : job.name ?? null))}
|
||||
aria-expanded={expanded === job.name}
|
||||
>
|
||||
<span className="font-mono text-xs font-semibold">{job.name}</span>
|
||||
<span className="text-xs text-muted-foreground">{describeJob(job)}</span>
|
||||
<span className="w-full truncate text-xs text-muted-foreground">{summaryOf(job)}</span>
|
||||
</button>
|
||||
{expanded === job.name && (
|
||||
<pre className="mt-2 max-h-72 overflow-auto whitespace-pre-wrap rounded bg-background p-2 text-[11px]">
|
||||
{job.prompt.join('\n')}
|
||||
</pre>
|
||||
)}
|
||||
</li>
|
||||
))}
|
||||
</ol>
|
||||
</section>
|
||||
)}
|
||||
</div>
|
||||
|
||||
<DialogFooter>
|
||||
<Button type="button" variant="outline" onClick={onClose} disabled={creating}>
|
||||
{running ? 'Hide' : 'Cancel'}
|
||||
</Button>
|
||||
{draft?.status === 'completed' && draft.jobs.length > 0 ? (
|
||||
<>
|
||||
<Button type="button" variant="outline" onClick={start} disabled={!canStart || creating}>
|
||||
Redraft
|
||||
</Button>
|
||||
<Button type="button" onClick={createAll} disabled={creating}>
|
||||
{creating ? 'Creating…' : `Create ${draft.jobs.length} pending job${draft.jobs.length === 1 ? '' : 's'}`}
|
||||
</Button>
|
||||
</>
|
||||
) : (
|
||||
<Button type="button" onClick={start} disabled={!canStart}>
|
||||
{starting ? 'Starting…' : running ? 'Drafting…' : 'Draft clips'}
|
||||
</Button>
|
||||
)}
|
||||
</DialogFooter>
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -69,6 +69,8 @@ export interface CreateJobRequest {
|
||||
workload_type?: string;
|
||||
job_type?: JobType;
|
||||
image_path?: string;
|
||||
/** Ref2VA reference media, numbered by type in this order (<Picture 1>, <Audio 1>, ...). */
|
||||
references?: { source: string; media_type: MediaType }[];
|
||||
data_path?: string;
|
||||
max_train_steps?: number;
|
||||
train_batch_size?: number;
|
||||
@@ -638,3 +640,118 @@ export function getDatasetMediaUrl(
|
||||
const baseApiUrl = getApiBaseUrl();
|
||||
return `${baseApiUrl}/datasets/${datasetId}/media/${encodeURIComponent(fileName)}`;
|
||||
}
|
||||
|
||||
// MARK: - Scene writer
|
||||
|
||||
export interface SceneDraftReference {
|
||||
path: string;
|
||||
media_type: MediaType;
|
||||
description: string;
|
||||
}
|
||||
|
||||
export interface SceneDraftRequest {
|
||||
script: string;
|
||||
/** Lowercase words joined by "-"; clips are named `<scene_name>-clip-NN`. */
|
||||
scene_name: string;
|
||||
references: SceneDraftReference[];
|
||||
instructions: string;
|
||||
/** Let the writer download reference photos from Wikimedia Commons for subjects with none. */
|
||||
find_references: boolean;
|
||||
}
|
||||
|
||||
/** A photo the writer downloaded, with where it came from so it can be checked and credited. */
|
||||
export interface FetchedReference {
|
||||
path: string;
|
||||
subject: string;
|
||||
query: string;
|
||||
title: string;
|
||||
/** The Wikimedia Commons file page. */
|
||||
page: string;
|
||||
license: string;
|
||||
artist: string;
|
||||
width: number;
|
||||
height: number;
|
||||
}
|
||||
|
||||
/** A drafted job, in the job-file shape: `prompt` is a list of lines. */
|
||||
export type DraftedJob = Omit<CreateJobRequest, "prompt"> & { prompt: string[] };
|
||||
|
||||
export interface SceneDraft {
|
||||
run_id: string;
|
||||
scene_name: string;
|
||||
status: "running" | "completed" | "failed";
|
||||
started_at: number;
|
||||
finished_at: number | null;
|
||||
/** Clips written so far (while running) or in the end; every one passed validation. */
|
||||
jobs: DraftedJob[];
|
||||
error: string | null;
|
||||
final_response?: string | null;
|
||||
fetched_references: FetchedReference[];
|
||||
}
|
||||
|
||||
/** URL of an image the writer downloaded for a draft. */
|
||||
export function getFetchedReferenceUrl(runId: string, ref: FetchedReference): string {
|
||||
const name = ref.path.split("/").pop() ?? "";
|
||||
return `${getApiBaseUrl()}/scene-writer/drafts/${runId}/references/${encodeURIComponent(name)}`;
|
||||
}
|
||||
|
||||
/** Start drafting a scene's jobs; poll `getSceneDraft` until it finishes. */
|
||||
export async function startSceneDraft(req: SceneDraftRequest): Promise<SceneDraft> {
|
||||
const response = await fetch(`${getApiBaseUrl()}/scene-writer/drafts`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(req),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({ detail: "Failed to start the draft" }));
|
||||
throw new Error(typeof error.detail === "string" ? error.detail : "Failed to start the draft");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function getSceneDraft(runId: string): Promise<SceneDraft> {
|
||||
const response = await fetch(`${getApiBaseUrl()}/scene-writer/drafts/${runId}`);
|
||||
if (!response.ok) {
|
||||
throw new Error("Failed to fetch the draft");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/** A clip revision: the agent's rewrite of one clip's job from feedback on its take. */
|
||||
export interface ClipRevision {
|
||||
run_id: string;
|
||||
status: "running" | "completed" | "failed";
|
||||
started_at: number;
|
||||
finished_at: number | null;
|
||||
/** the clip's job as it was sent, in job-file form */
|
||||
original: DraftedJob;
|
||||
/** the revised job; null until it is written */
|
||||
job: DraftedJob | null;
|
||||
/** the agent's explanation: the cause it found and each change it made */
|
||||
final_response: string | null;
|
||||
error: string | null;
|
||||
}
|
||||
|
||||
/** Start revising a clip; poll `getClipRevision` until it finishes. */
|
||||
export async function startClipRevision(req: {
|
||||
job: Job;
|
||||
feedback: string;
|
||||
neighbors: Job[];
|
||||
}): Promise<ClipRevision> {
|
||||
const response = await fetch(`${getApiBaseUrl()}/scene-writer/revisions`, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json" },
|
||||
body: JSON.stringify(req),
|
||||
});
|
||||
if (!response.ok) {
|
||||
const error = await response.json().catch(() => ({ detail: "Failed to start the revision" }));
|
||||
throw new Error(typeof error.detail === "string" ? error.detail : "Failed to start the revision");
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
export async function getClipRevision(runId: string): Promise<ClipRevision> {
|
||||
const response = await fetch(`${getApiBaseUrl()}/scene-writer/revisions/${runId}`);
|
||||
if (!response.ok) throw new Error("Failed to load the revision");
|
||||
return response.json();
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import { diffLines } from "./lineDiff";
|
||||
|
||||
describe("diffLines", () => {
|
||||
it("marks a changed line as removed then added, keeping the rest", () => {
|
||||
expect(diffLines("a\nslap on each four\nc", "a\nslap on the first four\nc")).toEqual([
|
||||
{ kind: "same", text: "a" },
|
||||
{ kind: "removed", text: "slap on each four" },
|
||||
{ kind: "added", text: "slap on the first four" },
|
||||
{ kind: "same", text: "c" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("handles added and removed lines at either end", () => {
|
||||
expect(diffLines("x\ny", "y\nz")).toEqual([
|
||||
{ kind: "removed", text: "x" },
|
||||
{ kind: "same", text: "y" },
|
||||
{ kind: "added", text: "z" },
|
||||
]);
|
||||
});
|
||||
|
||||
it("returns only kept lines for identical texts", () => {
|
||||
expect(diffLines("a\nb", "a\nb").every((l) => l.kind === "same")).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
/** One line of a line diff: kept, removed from the old text, or added in the new. */
|
||||
export interface DiffLine {
|
||||
kind: "same" | "removed" | "added";
|
||||
text: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Line-by-line diff of two texts (longest common subsequence), in reading
|
||||
* order: within a changed run, removed lines come before added ones.
|
||||
*/
|
||||
export function diffLines(oldText: string, newText: string): DiffLine[] {
|
||||
const a = oldText.split("\n");
|
||||
const b = newText.split("\n");
|
||||
// lcs[i][j]: length of the longest common subsequence of a[i:] and b[j:]
|
||||
const lcs = Array.from({ length: a.length + 1 }, () => new Array<number>(b.length + 1).fill(0));
|
||||
for (let i = a.length - 1; i >= 0; i--) {
|
||||
for (let j = b.length - 1; j >= 0; j--) {
|
||||
lcs[i][j] = a[i] === b[j] ? lcs[i + 1][j + 1] + 1 : Math.max(lcs[i + 1][j], lcs[i][j + 1]);
|
||||
}
|
||||
}
|
||||
const out: DiffLine[] = [];
|
||||
let i = 0;
|
||||
let j = 0;
|
||||
while (i < a.length || j < b.length) {
|
||||
if (i < a.length && j < b.length && a[i] === b[j]) {
|
||||
out.push({ kind: "same", text: a[i] });
|
||||
i++;
|
||||
j++;
|
||||
} else if (j >= b.length || (i < a.length && lcs[i + 1][j] >= lcs[i][j + 1])) {
|
||||
out.push({ kind: "removed", text: a[i] });
|
||||
i++;
|
||||
} else {
|
||||
out.push({ kind: "added", text: b[j] });
|
||||
j++;
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
@@ -0,0 +1,126 @@
|
||||
# SPDX-License-Identifier: Apache-2.0
|
||||
"""Scene writer pieces that run without a model: settings, job forms, request validation."""
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
from fastvideo_studio import mock_server
|
||||
from fastvideo_studio.scene_writer import runner
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def client():
|
||||
return TestClient(mock_server.app)
|
||||
|
||||
|
||||
def test_settings_file_values_apply_unless_the_environment_sets_them(tmp_path, monkeypatch):
|
||||
settings_file = tmp_path / "scene-writer.env"
|
||||
settings_file.write_text("# comment\nSCENE_WRITER_PROVIDER=cerebras\nexport SCENE_WRITER_MODEL='qwen-3.8-27b'\n"
|
||||
"SCENE_WRITER_API_KEY=\n")
|
||||
monkeypatch.setenv("SCENE_WRITER_SETTINGS_FILE", str(settings_file))
|
||||
for name in ("SCENE_WRITER_PROVIDER", "SCENE_WRITER_MODEL", "SCENE_WRITER_API_KEY"):
|
||||
monkeypatch.delenv(name, raising=False)
|
||||
|
||||
settings = runner.SceneWriterSettings()
|
||||
assert (settings.provider, settings.model, settings.api_key) == ("cerebras", "qwen-3.8-27b", "")
|
||||
|
||||
monkeypatch.setenv("SCENE_WRITER_MODEL", "gpt-oss-120b")
|
||||
assert runner.SceneWriterSettings().model == "gpt-oss-120b"
|
||||
|
||||
|
||||
def test_drafts_go_under_the_server_data_dir_unless_a_runs_root_is_set(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("SCENE_WRITER_SETTINGS_FILE", str(tmp_path / "none.env"))
|
||||
monkeypatch.delenv("SCENE_WRITER_RUNS_ROOT", raising=False)
|
||||
monkeypatch.setattr(runner, "_server_runs_root", None)
|
||||
assert runner.SceneWriterSettings().runs_root == Path.home() / "h3_scene_writer"
|
||||
|
||||
runner.use_server_data_dir(tmp_path / "ui_data")
|
||||
assert runner.SceneWriterSettings().runs_root == tmp_path / "ui_data" / "scene_writer"
|
||||
|
||||
monkeypatch.setenv("SCENE_WRITER_RUNS_ROOT", str(tmp_path / "elsewhere"))
|
||||
assert runner.SceneWriterSettings().runs_root == tmp_path / "elsewhere"
|
||||
|
||||
|
||||
def test_cerebras_defaults_to_its_model_and_routes_the_key_to_the_child():
|
||||
settings = runner.SceneWriterSettings(provider="cerebras", model=None, api_key="k", node_bin_dir=None)
|
||||
assert settings.resolved_model() == "qwen-3.8-27b"
|
||||
assert settings.child_env() == {runner.PI_AI_KEY_ENV: "k"}
|
||||
|
||||
|
||||
def test_a_model_missing_from_the_harness_catalog_is_declared_with_image_input(tmp_path):
|
||||
def patch_for(model):
|
||||
settings = runner.SceneWriterSettings(provider="cerebras", model=model, api_key="k", node_bin_dir=None)
|
||||
runner._write_patch(tmp_path / "p.yml", tmp_path / "jobs", [], [], None, settings)
|
||||
return (tmp_path / "p.yml").read_text()
|
||||
|
||||
assert '"id": "qwen-3.8-27b"' in patch_for("qwen-3.8-27b")
|
||||
assert '"input": ["text", "image"]' in patch_for("qwen-3.8-27b")
|
||||
assert f"requestImagePixelBudget: {runner.IMAGE_PIXEL_BUDGET}" in patch_for("qwen-3.8-27b")
|
||||
# A catalog model keeps the catalog's own entry.
|
||||
assert "models:" not in patch_for("gpt-oss-120b")
|
||||
|
||||
|
||||
def test_take_stills_are_spread_over_the_clip_and_named_by_time(tmp_path):
|
||||
av = pytest.importorskip("av")
|
||||
import numpy as np
|
||||
|
||||
video = tmp_path / "take.mp4"
|
||||
with av.open(str(video), "w") as container:
|
||||
stream = container.add_stream("libx264", rate=24)
|
||||
stream.width, stream.height, stream.pix_fmt = 320, 180, "yuv420p"
|
||||
for i in range(24 * 6): # 6 s
|
||||
frame = av.VideoFrame.from_ndarray(np.full((180, 320, 3), i % 255, np.uint8), format="rgb24")
|
||||
container.mux(stream.encode(frame))
|
||||
container.mux(stream.encode())
|
||||
|
||||
stills = runner.extract_frames(str(video), tmp_path / "frames", width=160)
|
||||
assert [p.name for p in stills] == ["t01.0s.jpg", "t03.0s.jpg", "t05.0s.jpg"]
|
||||
assert runner.extract_frames(str(tmp_path / "missing.mp4"), tmp_path / "none") == []
|
||||
|
||||
|
||||
def test_studio_job_becomes_a_job_file_without_runtime_state():
|
||||
studio_job = {
|
||||
"id": "abc", "status": "completed", "output_path": "/out.mp4", "name": "s-clip-01",
|
||||
"model_id": "MiniMaxAI/MiniMax-H3", "prompt": "line one\nline two", "num_frames": 48, "fps": 24,
|
||||
"references": [{"source": "/a.png", "media_type": "image", "extra": 1}], "negative_prompt": None,
|
||||
}
|
||||
assert runner.job_file_form(studio_job) == {
|
||||
"name": "s-clip-01", "model_id": "MiniMaxAI/MiniMax-H3", "prompt": ["line one", "line two"],
|
||||
"num_frames": 48, "fps": 24, "references": [{"source": "/a.png", "media_type": "image"}],
|
||||
}
|
||||
|
||||
|
||||
def test_a_draft_needs_references_or_permission_to_find_them(client):
|
||||
body = {"script": "x", "scene_name": "a", "references": [], "instructions": ""}
|
||||
assert client.post("/api/scene-writer/drafts", json=body).status_code == 422
|
||||
|
||||
|
||||
def test_a_revision_needs_feedback_and_a_named_job(client):
|
||||
assert client.post("/api/scene-writer/revisions", json={"job": {"name": "a-clip-01"}, "feedback": ""}).status_code == 422
|
||||
assert client.post("/api/scene-writer/revisions", json={"job": {}, "feedback": "too many slaps"}).status_code == 422
|
||||
assert client.get("/api/scene-writer/revisions/missing").status_code == 404
|
||||
|
||||
|
||||
def test_mock_duplicate_makes_a_pending_take_with_the_same_config(client):
|
||||
job = client.post("/api/jobs", json={"model_id": "MiniMaxAI/MiniMax-H3", "name": "t-clip-01", "prompt": "p",
|
||||
"num_frames": 48,
|
||||
"references": [{"source": "/x.png", "media_type": "image"}]}).json()
|
||||
take = client.post(f"/api/jobs/{job['id']}/duplicate").json()
|
||||
|
||||
assert take["id"] != job["id"]
|
||||
assert (take["name"], take["status"], take["references"], take["num_frames"]) == (
|
||||
"t-clip-01", "pending", job["references"], 48)
|
||||
assert client.post("/api/jobs/missing/duplicate").status_code == 404
|
||||
|
||||
|
||||
def test_a_provider_error_names_the_cause():
|
||||
settings = runner.SceneWriterSettings(provider="cerebras", api_key="k", node_bin_dir=None)
|
||||
|
||||
def ended_with(message):
|
||||
return [{"type": "turn/end", "data": {"reason": {"kind": "error", "error": {"message": message}}}}]
|
||||
|
||||
assert "402 Payment Required" in runner._turn_error_message(ended_with("402 status code (no body)"), settings)
|
||||
assert "rejected SCENE_WRITER_API_KEY" in runner._turn_error_message(ended_with("401 Unauthorized"), settings)
|
||||
assert runner._turn_error_message(ended_with("socket hang up"), settings) == "The model call failed: socket hang up"
|
||||
Reference in New Issue
Block a user