feat(inspect): bounded browser video tag readers
Add web/js/workspace/local-video.mjs: ISO-BMFF (MP4/MOV/M4V) and Matroska (MKV/WebM) walkers that read only the container tags the server asks ffprobe for, slicing box and element headers, carrier names and retained values within the server's budgets and never a byte inside mdat or a Cluster; a port of utils/video_metadata.select_video_carrier with the record budgets of read_inspection_file; and a byte-exact json.loads/json.dumps round trip so envelope objects reserialize as the server spells them. Every naming rule mirrors ffprobe 9.0.2 on hand-built files: legacy ilst atoms, mdta keys, joined repeats, data types, Matroska language and default handling, and Tags behind a Cluster found only through a SeekHead. Tests: node units over byte-exact container builders and the committed fixtures; a Python parity test comparing the local carrier with the server's selection from the same tag maps (mocked probe) and from the same files (real ffprobe, including 1 MiB + 2.5 MiB per-key records); an ffprobe characterization of the recorded fixture maps.
This commit is contained in:
Vito Sansevero
Vendored
+39
@@ -28,6 +28,45 @@ ffmpeg -f lavfi -i testsrc=size=48x32:rate=1 -frames:v 1 -q:v 10 tiny.jpg
|
||||
ffmpeg -f lavfi -i testsrc=size=40x24:rate=1 -frames:v 1 -c:v libwebp -quality 40 tiny.webp
|
||||
```
|
||||
|
||||
# Local Inspect video fixtures
|
||||
|
||||
`local-inspect/core-faststart.mp4`, `core-moov-end.mp4`, `core.mov`, `core.m4v`,
|
||||
`core.mkv`, `core-tags.webm`, `legacy-comment.mp4` and `legacy-comment.webm` are
|
||||
0.2-second 16x16 black clips (each under 4 KB) carrying an original synthetic
|
||||
ComfyUI graph in their container tags, dedicated to the public domain under
|
||||
CC0-1.0. The `core-*` files carry one tag per key as core SaveVideo/SaveWEBM write
|
||||
them (MP4/MOV/M4V through `-movflags use_metadata_tags`; Matroska upper-cases the
|
||||
names); `core-faststart.mp4` also carries `parameters` and
|
||||
`comfyassets_prompt_pairs`. The `legacy-comment.*` files carry the old
|
||||
VideoHelperSuite `comment` JSON envelope (an ilst `©cmt` atom in MP4, a `COMMENT`
|
||||
tag in WebM). `video-tags-v1.json` records `ffprobe -show_entries format_tags -of
|
||||
json` for each file (ffmpeg 9.0.2); `tests/test_utils_video_metadata.py`
|
||||
`VideoFixtureFfprobeTests` re-checks it when ffprobe is installed, and
|
||||
`tests/js/workspace-local-video.test.js` plus `tests/test_local_inspect_parity.py`
|
||||
hold the browser video reader to the same maps.
|
||||
|
||||
Reproduce with FFmpeg, from `tests/fixtures/local-inspect`:
|
||||
|
||||
```sh
|
||||
PROMPT='{"3":{"class_type":"KSampler","inputs":{"seed":7,"steps":20,"cfg":8.0,"sampler_name":"euler","scheduler":"normal","denoise":1.0,"model":["4",0],"positive":["6",0],"negative":["7",0],"latent_image":["5",0]}},"4":{"class_type":"CheckpointLoaderSimple","inputs":{"ckpt_name":"synthetic-v1.safetensors"}},"5":{"class_type":"EmptyLatentImage","inputs":{"width":512,"height":512,"batch_size":1}},"6":{"class_type":"CLIPTextEncode","inputs":{"text":"a red fox in snow, detailed fur","clip":["4",1]}},"7":{"class_type":"CLIPTextEncode","inputs":{"text":"blurry, lowres","clip":["4",1]}},"8":{"class_type":"VAEDecode","inputs":{"samples":["3",0],"vae":["4",2]}},"9":{"class_type":"SaveVideo","inputs":{"filename_prefix":"video/synthetic","video":["8",0]}}}'
|
||||
WORKFLOW='{"last_node_id":9,"nodes":[{"id":3,"type":"KSampler","widgets_values":[7,"fixed",20,8.0,"euler","normal",1.0]},{"id":6,"type":"CLIPTextEncode","widgets_values":["a red fox in snow, detailed fur"]},{"id":7,"type":"CLIPTextEncode","widgets_values":["blurry, lowres"]}],"links":[],"version":0.4}'
|
||||
PARAMETERS=$'a red fox in snow, detailed fur\nNegative prompt: blurry, lowres\nSteps: 20, Sampler: Euler, CFG scale: 8.0, Seed: 7, Size: 512x512'
|
||||
PAIRS='{"version":1,"nodes":{"6":{"records":[{"positive_text":"a red fox in snow, detailed fur","negative_text":"blurry, lowres","negative_effective_text":"blurry, lowres","prepend_text":"","append_text":""}]}}}'
|
||||
COMMENT=$(PROMPT="$PROMPT" WORKFLOW="$WORKFLOW" python3 -c 'import json,os; print(json.dumps({"prompt": os.environ["PROMPT"], "workflow": json.loads(os.environ["WORKFLOW"])}))')
|
||||
SRC=(-v quiet -y -f lavfi -i "color=c=black:s=16x16:d=0.2")
|
||||
ffmpeg "${SRC[@]}" -movflags use_metadata_tags+faststart -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" -metadata parameters="$PARAMETERS" -metadata comfyassets_prompt_pairs="$PAIRS" core-faststart.mp4
|
||||
ffmpeg "${SRC[@]}" -movflags use_metadata_tags -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core-moov-end.mp4
|
||||
ffmpeg "${SRC[@]}" -movflags use_metadata_tags -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core.mov
|
||||
ffmpeg "${SRC[@]}" -movflags use_metadata_tags -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core.m4v
|
||||
ffmpeg "${SRC[@]}" -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core.mkv
|
||||
ffmpeg "${SRC[@]}" -c:v libvpx -metadata prompt="$PROMPT" -metadata workflow="$WORKFLOW" core-tags.webm
|
||||
ffmpeg "${SRC[@]}" -metadata comment="$COMMENT" legacy-comment.mp4
|
||||
ffmpeg "${SRC[@]}" -c:v libvpx -metadata comment="$COMMENT" legacy-comment.webm
|
||||
for f in core-faststart.mp4 core-moov-end.mp4 core.mov core.m4v core.mkv core-tags.webm legacy-comment.mp4 legacy-comment.webm; do
|
||||
ffprobe -v quiet -show_entries format_tags -of json "$f" # -> video-tags-v1.json "fixtures"[f] = .format.tags
|
||||
done
|
||||
```
|
||||
|
||||
# Inspection parity fixtures
|
||||
|
||||
`inspection_parity/` holds small synthetic PNGs (`f01b.png` ... `f24.png`, each
|
||||
|
||||
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
BIN
Binary file not shown.
Binary file not shown.
+60
@@ -0,0 +1,60 @@
|
||||
{
|
||||
"generator": "ffmpeg -show_entries format_tags -of json over tests/fixtures/local-inspect, see tests/fixtures/README.md",
|
||||
"fixtures": {
|
||||
"core-faststart.mp4": {
|
||||
"major_brand": "isom",
|
||||
"minor_version": "512",
|
||||
"compatible_brands": "isomiso2avc1mp41",
|
||||
"prompt": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}",
|
||||
"workflow": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}",
|
||||
"parameters": "a red fox in snow, detailed fur\nNegative prompt: blurry, lowres\nSteps: 20, Sampler: Euler, CFG scale: 8.0, Seed: 7, Size: 512x512",
|
||||
"comfyassets_prompt_pairs": "{\"version\":1,\"nodes\":{\"6\":{\"records\":[{\"positive_text\":\"a red fox in snow, detailed fur\",\"negative_text\":\"blurry, lowres\",\"negative_effective_text\":\"blurry, lowres\",\"prepend_text\":\"\",\"append_text\":\"\"}]}}}",
|
||||
"encoder": "Lavf63.1.102"
|
||||
},
|
||||
"core-moov-end.mp4": {
|
||||
"major_brand": "isom",
|
||||
"minor_version": "512",
|
||||
"compatible_brands": "isomiso2avc1mp41",
|
||||
"prompt": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}",
|
||||
"workflow": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}",
|
||||
"encoder": "Lavf63.1.102"
|
||||
},
|
||||
"core.mov": {
|
||||
"major_brand": "qt ",
|
||||
"minor_version": "512",
|
||||
"compatible_brands": "qt ",
|
||||
"prompt": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}",
|
||||
"workflow": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}",
|
||||
"encoder": "Lavf63.1.102"
|
||||
},
|
||||
"core.m4v": {
|
||||
"major_brand": "M4V ",
|
||||
"minor_version": "512",
|
||||
"compatible_brands": "M4V isomiso2avc1",
|
||||
"prompt": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}",
|
||||
"workflow": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}",
|
||||
"encoder": "Lavf63.1.102"
|
||||
},
|
||||
"core.mkv": {
|
||||
"WORKFLOW": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}",
|
||||
"PROMPT": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}",
|
||||
"ENCODER": "Lavf63.1.102"
|
||||
},
|
||||
"core-tags.webm": {
|
||||
"WORKFLOW": "{\"last_node_id\":9,\"nodes\":[{\"id\":3,\"type\":\"KSampler\",\"widgets_values\":[7,\"fixed\",20,8.0,\"euler\",\"normal\",1.0]},{\"id\":6,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"a red fox in snow, detailed fur\"]},{\"id\":7,\"type\":\"CLIPTextEncode\",\"widgets_values\":[\"blurry, lowres\"]}],\"links\":[],\"version\":0.4}",
|
||||
"PROMPT": "{\"3\":{\"class_type\":\"KSampler\",\"inputs\":{\"seed\":7,\"steps\":20,\"cfg\":8.0,\"sampler_name\":\"euler\",\"scheduler\":\"normal\",\"denoise\":1.0,\"model\":[\"4\",0],\"positive\":[\"6\",0],\"negative\":[\"7\",0],\"latent_image\":[\"5\",0]}},\"4\":{\"class_type\":\"CheckpointLoaderSimple\",\"inputs\":{\"ckpt_name\":\"synthetic-v1.safetensors\"}},\"5\":{\"class_type\":\"EmptyLatentImage\",\"inputs\":{\"width\":512,\"height\":512,\"batch_size\":1}},\"6\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"a red fox in snow, detailed fur\",\"clip\":[\"4\",1]}},\"7\":{\"class_type\":\"CLIPTextEncode\",\"inputs\":{\"text\":\"blurry, lowres\",\"clip\":[\"4\",1]}},\"8\":{\"class_type\":\"VAEDecode\",\"inputs\":{\"samples\":[\"3\",0],\"vae\":[\"4\",2]}},\"9\":{\"class_type\":\"SaveVideo\",\"inputs\":{\"filename_prefix\":\"video/synthetic\",\"video\":[\"8\",0]}}}",
|
||||
"ENCODER": "Lavf63.1.102"
|
||||
},
|
||||
"legacy-comment.mp4": {
|
||||
"major_brand": "isom",
|
||||
"minor_version": "512",
|
||||
"compatible_brands": "isomiso2avc1mp41",
|
||||
"encoder": "Lavf63.1.102",
|
||||
"comment": "{\"prompt\": \"{\\\"3\\\":{\\\"class_type\\\":\\\"KSampler\\\",\\\"inputs\\\":{\\\"seed\\\":7,\\\"steps\\\":20,\\\"cfg\\\":8.0,\\\"sampler_name\\\":\\\"euler\\\",\\\"scheduler\\\":\\\"normal\\\",\\\"denoise\\\":1.0,\\\"model\\\":[\\\"4\\\",0],\\\"positive\\\":[\\\"6\\\",0],\\\"negative\\\":[\\\"7\\\",0],\\\"latent_image\\\":[\\\"5\\\",0]}},\\\"4\\\":{\\\"class_type\\\":\\\"CheckpointLoaderSimple\\\",\\\"inputs\\\":{\\\"ckpt_name\\\":\\\"synthetic-v1.safetensors\\\"}},\\\"5\\\":{\\\"class_type\\\":\\\"EmptyLatentImage\\\",\\\"inputs\\\":{\\\"width\\\":512,\\\"height\\\":512,\\\"batch_size\\\":1}},\\\"6\\\":{\\\"class_type\\\":\\\"CLIPTextEncode\\\",\\\"inputs\\\":{\\\"text\\\":\\\"a red fox in snow, detailed fur\\\",\\\"clip\\\":[\\\"4\\\",1]}},\\\"7\\\":{\\\"class_type\\\":\\\"CLIPTextEncode\\\",\\\"inputs\\\":{\\\"text\\\":\\\"blurry, lowres\\\",\\\"clip\\\":[\\\"4\\\",1]}},\\\"8\\\":{\\\"class_type\\\":\\\"VAEDecode\\\",\\\"inputs\\\":{\\\"samples\\\":[\\\"3\\\",0],\\\"vae\\\":[\\\"4\\\",2]}},\\\"9\\\":{\\\"class_type\\\":\\\"SaveVideo\\\",\\\"inputs\\\":{\\\"filename_prefix\\\":\\\"video/synthetic\\\",\\\"video\\\":[\\\"8\\\",0]}}}\", \"workflow\": {\"last_node_id\": 9, \"nodes\": [{\"id\": 3, \"type\": \"KSampler\", \"widgets_values\": [7, \"fixed\", 20, 8.0, \"euler\", \"normal\", 1.0]}, {\"id\": 6, \"type\": \"CLIPTextEncode\", \"widgets_values\": [\"a red fox in snow, detailed fur\"]}, {\"id\": 7, \"type\": \"CLIPTextEncode\", \"widgets_values\": [\"blurry, lowres\"]}], \"links\": [], \"version\": 0.4}}"
|
||||
},
|
||||
"legacy-comment.webm": {
|
||||
"COMMENT": "{\"prompt\": \"{\\\"3\\\":{\\\"class_type\\\":\\\"KSampler\\\",\\\"inputs\\\":{\\\"seed\\\":7,\\\"steps\\\":20,\\\"cfg\\\":8.0,\\\"sampler_name\\\":\\\"euler\\\",\\\"scheduler\\\":\\\"normal\\\",\\\"denoise\\\":1.0,\\\"model\\\":[\\\"4\\\",0],\\\"positive\\\":[\\\"6\\\",0],\\\"negative\\\":[\\\"7\\\",0],\\\"latent_image\\\":[\\\"5\\\",0]}},\\\"4\\\":{\\\"class_type\\\":\\\"CheckpointLoaderSimple\\\",\\\"inputs\\\":{\\\"ckpt_name\\\":\\\"synthetic-v1.safetensors\\\"}},\\\"5\\\":{\\\"class_type\\\":\\\"EmptyLatentImage\\\",\\\"inputs\\\":{\\\"width\\\":512,\\\"height\\\":512,\\\"batch_size\\\":1}},\\\"6\\\":{\\\"class_type\\\":\\\"CLIPTextEncode\\\",\\\"inputs\\\":{\\\"text\\\":\\\"a red fox in snow, detailed fur\\\",\\\"clip\\\":[\\\"4\\\",1]}},\\\"7\\\":{\\\"class_type\\\":\\\"CLIPTextEncode\\\",\\\"inputs\\\":{\\\"text\\\":\\\"blurry, lowres\\\",\\\"clip\\\":[\\\"4\\\",1]}},\\\"8\\\":{\\\"class_type\\\":\\\"VAEDecode\\\",\\\"inputs\\\":{\\\"samples\\\":[\\\"3\\\",0],\\\"vae\\\":[\\\"4\\\",2]}},\\\"9\\\":{\\\"class_type\\\":\\\"SaveVideo\\\",\\\"inputs\\\":{\\\"filename_prefix\\\":\\\"video/synthetic\\\",\\\"video\\\":[\\\"8\\\",0]}}}\", \"workflow\": {\"last_node_id\": 9, \"nodes\": [{\"id\": 3, \"type\": \"KSampler\", \"widgets_values\": [7, \"fixed\", 20, 8.0, \"euler\", \"normal\", 1.0]}, {\"id\": 6, \"type\": \"CLIPTextEncode\", \"widgets_values\": [\"a red fox in snow, detailed fur\"]}, {\"id\": 7, \"type\": \"CLIPTextEncode\", \"widgets_values\": [\"blurry, lowres\"]}], \"links\": [], \"version\": 0.4}}",
|
||||
"ENCODER": "Lavf63.1.102"
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
// Byte-exact ISO-BMFF (MP4/MOV/M4V) and Matroska (MKV/WebM) builders for the local
|
||||
// video tag reader tests, laid out as ffmpeg's muxers write them (see the dumps in
|
||||
// tests/fixtures/local-inspect). The File-like blobs come from png-bytes.mjs.
|
||||
import { concat, u32, utf8, latin1 } from "./png-bytes.mjs";
|
||||
|
||||
export { concat, u32, utf8, latin1 };
|
||||
export { sparseBlob, recordingBlob, slowBlob, failingBlob, blobOf } from "./png-bytes.mjs";
|
||||
|
||||
const toBytes = (part) => (typeof part === "string" ? utf8(part) : Buffer.from(part));
|
||||
|
||||
function u64(n) {
|
||||
const out = new Uint8Array(8);
|
||||
new DataView(out.buffer).setBigUint64(0, BigInt(n));
|
||||
return out;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- ISO-BMFF
|
||||
|
||||
/** A box: 32-bit size, 4-byte type (string or raw bytes), payload. `size` overrides the field. */
|
||||
export function box(type, ...parts) {
|
||||
const options = parts.length && parts[parts.length - 1] && parts[parts.length - 1].__options ? parts.pop() : {};
|
||||
const payload = concat(...parts.map(toBytes));
|
||||
const typeBytes = typeof type === "string" ? latin1(type) : Buffer.from(type);
|
||||
const total = 8 + payload.length;
|
||||
if (options.largesize) return concat(u32(1), typeBytes, u64(options.largesize === true ? total + 8 : options.largesize), payload);
|
||||
return concat(u32(options.size ?? total), typeBytes, payload);
|
||||
}
|
||||
|
||||
/** Marker for `box` options: box('ftyp', ..., opts({size: 3})). */
|
||||
export function opts(options) {
|
||||
return { __options: true, ...options };
|
||||
}
|
||||
|
||||
export function fullBox(type, ...parts) {
|
||||
return box(type, u32(0), ...parts);
|
||||
}
|
||||
|
||||
export function ftyp(brand = "isom", compatible = "isomiso2avc1mp41") {
|
||||
return box("ftyp", latin1(brand), u32(512), latin1(compatible));
|
||||
}
|
||||
|
||||
export function mdat(payload = [0, 0, 0, 0]) {
|
||||
return box("mdat", payload);
|
||||
}
|
||||
|
||||
export const moov = (...children) => box("moov", ...children);
|
||||
export const udta = (...children) => box("udta", ...children);
|
||||
export const free = (count = 0) => box("free", new Uint8Array(count));
|
||||
|
||||
/** `meta` as ffmpeg writes it: a FullBox (version/flags) whose first child is hdlr. */
|
||||
export function meta(...children) {
|
||||
return fullBox("meta", ...children);
|
||||
}
|
||||
|
||||
export function hdlr(handler = "mdta") {
|
||||
return fullBox("hdlr", u32(0), latin1(handler), new Uint8Array(12), [0]);
|
||||
}
|
||||
|
||||
/** `keys`: one mdta entry per name (an entry may be [name, namespace]); `count` overrides the count field. */
|
||||
export function keys(names, { count } = {}) {
|
||||
const entries = names.map((entry) => {
|
||||
const [name, namespace] = Array.isArray(entry) ? entry : [entry, "mdta"];
|
||||
const bytes = toBytes(name);
|
||||
return concat(u32(8 + bytes.length), latin1(namespace), bytes);
|
||||
});
|
||||
return fullBox("keys", u32(count ?? names.length), ...entries);
|
||||
}
|
||||
|
||||
export const ilst = (...items) => box("ilst", ...items);
|
||||
|
||||
/** A `data` atom: type indicator (1 = UTF-8), locale, payload. */
|
||||
export function data(payload, { type = 1, locale = 0, size } = {}) {
|
||||
return box("data", u32(type), u32(locale), toBytes(payload), ...(size === undefined ? [] : [opts({ size })]));
|
||||
}
|
||||
|
||||
/** An ilst item addressed by its 1-based `keys` index. */
|
||||
export function item(index, ...children) {
|
||||
return box(u32(index), ...children);
|
||||
}
|
||||
|
||||
/** An ilst item addressed by a legacy fourcc such as "©cmt" (Latin-1 bytes). */
|
||||
export function legacyItem(fourcc, ...children) {
|
||||
return box(latin1(fourcc), ...children);
|
||||
}
|
||||
|
||||
/** `moov/udta/meta` carrying per-key mdta tags, as core SaveVideo writes through ffmpeg. */
|
||||
export function mdtaUdta(tags) {
|
||||
const names = Object.keys(tags);
|
||||
return udta(meta(hdlr("mdta"), keys(names), ilst(...names.map((name, index) => item(index + 1, data(tags[name]))))));
|
||||
}
|
||||
|
||||
/** `moov/udta/meta` with legacy ilst atoms: {comment, description} → ©cmt / desc. */
|
||||
export function legacyUdta({ comment, description, tool = "Lavf" } = {}) {
|
||||
const items = [legacyItem("©too", data(tool))];
|
||||
if (comment !== undefined) items.push(legacyItem("©cmt", data(comment)));
|
||||
if (description !== undefined) items.push(legacyItem("desc", data(description)));
|
||||
return udta(meta(hdlr("mdir"), ilst(...items)));
|
||||
}
|
||||
|
||||
/** A whole MP4: ftyp, (moov first when faststart), mdat, moov with the given udta. */
|
||||
export function mp4(udtaBox, { brand = "isom", faststart = false, extraMoov = [], payload } = {}) {
|
||||
const movie = moov(...extraMoov, udtaBox);
|
||||
const media = mdat(payload);
|
||||
return faststart ? concat(ftyp(brand), movie, media) : concat(ftyp(brand), free(), media, movie);
|
||||
}
|
||||
|
||||
export function perKeyMp4(tags, options) {
|
||||
return mp4(mdtaUdta(tags), options);
|
||||
}
|
||||
|
||||
/** Top-level boxes of `bytes` as [{type, start, end}]. */
|
||||
export function topLevelBoxes(bytes) {
|
||||
const out = [];
|
||||
let offset = 0;
|
||||
while (offset + 8 <= bytes.length) {
|
||||
const view = new DataView(bytes.buffer, bytes.byteOffset + offset);
|
||||
let size = view.getUint32(0);
|
||||
if (size === 1) size = Number(view.getBigUint64(8));
|
||||
if (size === 0) size = bytes.length - offset;
|
||||
out.push({ type: latin1Of(bytes.subarray(offset + 4, offset + 8)), start: offset, end: offset + size });
|
||||
offset += size;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function latin1Of(bytes) {
|
||||
return Buffer.from(bytes).toString("latin1");
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------- Matroska
|
||||
|
||||
export const ID = {
|
||||
EBML: 0x1a45dfa3,
|
||||
EBMLVersion: 0x4286,
|
||||
DocType: 0x4282,
|
||||
DocTypeVersion: 0x4287,
|
||||
Segment: 0x18538067,
|
||||
SeekHead: 0x114d9b74,
|
||||
Seek: 0x4dbb,
|
||||
SeekID: 0x53ab,
|
||||
SeekPosition: 0x53ac,
|
||||
Void: 0xec,
|
||||
CRC32: 0xbf,
|
||||
Info: 0x1549a966,
|
||||
Tracks: 0x1654ae6b,
|
||||
Tags: 0x1254c367,
|
||||
Tag: 0x7373,
|
||||
Targets: 0x63c0,
|
||||
TargetTypeValue: 0x68ca,
|
||||
TagTrackUID: 0x63c5,
|
||||
SimpleTag: 0x67c8,
|
||||
TagName: 0x45a3,
|
||||
TagString: 0x4487,
|
||||
TagLanguage: 0x447a,
|
||||
TagDefault: 0x4484,
|
||||
TagBinary: 0x4485,
|
||||
Cluster: 0x1f43b675,
|
||||
Timestamp: 0xe7,
|
||||
Cues: 0x1c53bb6b,
|
||||
};
|
||||
|
||||
/** The ID's bytes (EBML IDs carry their own length marker). */
|
||||
export function idBytes(id) {
|
||||
const out = [];
|
||||
let value = id;
|
||||
while (value > 0) {
|
||||
out.unshift(value & 0xff);
|
||||
value = Math.floor(value / 256);
|
||||
}
|
||||
return Uint8Array.from(out);
|
||||
}
|
||||
|
||||
/** A size VINT: minimal width unless `width` is given; `unknown` writes all ones. */
|
||||
export function vint(value, { width, unknown = false } = {}) {
|
||||
let length = width;
|
||||
if (!length) {
|
||||
length = 1;
|
||||
while (length < 8 && value >= 2 ** (7 * length) - 1) length += 1;
|
||||
}
|
||||
const out = new Uint8Array(length);
|
||||
if (unknown) {
|
||||
out.fill(0xff);
|
||||
out[0] = 0xff >> (length - 1);
|
||||
return out;
|
||||
}
|
||||
let rest = BigInt(value);
|
||||
for (let index = length - 1; index >= 0; index -= 1) {
|
||||
out[index] = Number(rest & 0xffn);
|
||||
rest >>= 8n;
|
||||
}
|
||||
out[0] |= 0x80 >> (length - 1);
|
||||
return out;
|
||||
}
|
||||
|
||||
/** An element: ID, size VINT, payload. `size`/`width`/`unknown` shape the size field. */
|
||||
export function el(id, ...parts) {
|
||||
const options = parts.length && parts[parts.length - 1] && parts[parts.length - 1].__options ? parts.pop() : {};
|
||||
const payload = concat(...parts.map(toBytes));
|
||||
const size = vint(options.size ?? payload.length, { width: options.width, unknown: options.unknown });
|
||||
return concat(idBytes(id), size, payload);
|
||||
}
|
||||
|
||||
export function uint(value, width) {
|
||||
const out = [];
|
||||
let rest = BigInt(value);
|
||||
do {
|
||||
out.unshift(Number(rest & 0xffn));
|
||||
rest >>= 8n;
|
||||
} while (rest > 0n);
|
||||
while (width && out.length < width) out.unshift(0);
|
||||
return Uint8Array.from(out);
|
||||
}
|
||||
|
||||
export function ebmlHeader(docType = "matroska") {
|
||||
return el(ID.EBML, el(ID.EBMLVersion, [1]), el(ID.DocType, docType), el(ID.DocTypeVersion, [4]));
|
||||
}
|
||||
|
||||
/** Segment with an 8-byte size field as ffmpeg writes it; `unknown` makes it unknown-sized. */
|
||||
export function segment(children, { unknown = false } = {}) {
|
||||
return el(ID.Segment, ...children, opts({ width: 8, unknown }));
|
||||
}
|
||||
|
||||
/** SeekHead entries: [[id, position]] with positions relative to the Segment payload. */
|
||||
export function seekHead(entries) {
|
||||
return el(ID.SeekHead, ...entries.map(([id, position]) => el(ID.Seek, el(ID.SeekID, idBytes(id)), el(ID.SeekPosition, uint(position)))));
|
||||
}
|
||||
|
||||
export function voidEl(count = 4) {
|
||||
return el(ID.Void, new Uint8Array(count));
|
||||
}
|
||||
|
||||
/** A SimpleTag with TagName and TagString (`value` undefined omits the string). */
|
||||
export function simpleTag(name, value, { lang, def, binary, nested = [], extra = [] } = {}) {
|
||||
const children = [el(ID.TagName, name)];
|
||||
if (lang !== undefined) children.push(el(ID.TagLanguage, lang));
|
||||
if (def !== undefined) children.push(el(ID.TagDefault, [def]));
|
||||
if (value !== undefined) children.push(el(ID.TagString, value));
|
||||
if (binary !== undefined) children.push(el(ID.TagBinary, binary));
|
||||
return el(ID.SimpleTag, ...children, ...nested, ...extra);
|
||||
}
|
||||
|
||||
/** A Tag whose Targets carry the given track UID (none → global). */
|
||||
export function tag(simpleTags, { trackUid, targetType } = {}) {
|
||||
const targets = [];
|
||||
if (targetType !== undefined) targets.push(el(ID.TargetTypeValue, uint(targetType)));
|
||||
if (trackUid !== undefined) targets.push(el(ID.TagTrackUID, uint(trackUid, 8)));
|
||||
return el(ID.Tag, el(ID.Targets, ...targets), ...simpleTags);
|
||||
}
|
||||
|
||||
export function tagsEl(...tagElements) {
|
||||
return el(ID.Tags, ...tagElements);
|
||||
}
|
||||
|
||||
/** A Cluster of `size` payload bytes (zeros); `unknown` makes its size unknown. */
|
||||
export function cluster(size = 8, { unknown = false, payload } = {}) {
|
||||
return el(ID.Cluster, payload ?? new Uint8Array(size), opts({ unknown }));
|
||||
}
|
||||
|
||||
/** A Cluster header only, declaring `size` payload bytes (for sparse layouts). */
|
||||
export function clusterHeader(size) {
|
||||
return concat(idBytes(ID.Cluster), vint(size, { width: 8 }));
|
||||
}
|
||||
|
||||
/** A whole Matroska file: EBML header + Segment(children). */
|
||||
export function mkv(children, { docType = "matroska", unknown = false } = {}) {
|
||||
return concat(ebmlHeader(docType), segment(children, { unknown }));
|
||||
}
|
||||
|
||||
/** A global Tags element from {name: value} (ffmpeg upper-cases names at mux time). */
|
||||
export function globalTags(tags, options) {
|
||||
return tagsEl(tag(Object.entries(tags).map(([name, value]) => simpleTag(name, value)), options));
|
||||
}
|
||||
|
||||
/** A Matroska file whose global Tags hold {name: value}, ffmpeg layout. */
|
||||
export function perKeyMkv(tags, { docType = "matroska" } = {}) {
|
||||
return mkv([el(ID.Info, new Uint8Array(4)), globalTags(tags), cluster()], { docType });
|
||||
}
|
||||
@@ -2,9 +2,13 @@
|
||||
// tests/test_local_inspect_parity.py, which compares every answer with the server's
|
||||
// utils.prompt_inspector.read_inspection_file on the same file. Each file is opened
|
||||
// as a Blob (fs.openAsBlob) so the reader slices it exactly as it slices a File.
|
||||
// stdin: [{name, path}]
|
||||
// stdin: [{name, path} | {name, tags, limited?}]
|
||||
// stdout: {name: {classified, result}} where result is readPngMetadataFile's
|
||||
// answer for a PNG, readImageHeader's for another image, else null.
|
||||
// answer for a PNG, readImageHeader's for another image, for a video
|
||||
// {read: readVideoTags(...), selection: selectVideoCarrier(read.tags, {limited})}
|
||||
// (selection null unless the read completed), else null.
|
||||
// A {tags} entry answers {classified: null, result: {selection}} for the
|
||||
// carrier rule alone, as the server selects from ffprobe's tag map.
|
||||
import fs from 'node:fs';
|
||||
import path from 'node:path';
|
||||
import { createRequire } from 'node:module';
|
||||
@@ -13,17 +17,27 @@ import { fileURLToPath } from 'node:url';
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const require = createRequire(import.meta.url);
|
||||
const { classifyPrefix, readPngMetadataFile, readImageHeader } = require(path.join(here, '..', '..', '..', 'web', 'js', 'png-metadata.js'));
|
||||
const { readVideoTags, selectVideoCarrier } = await import(path.join(here, '..', '..', '..', 'web', 'js', 'workspace', 'local-video.mjs'));
|
||||
|
||||
const chunks = [];
|
||||
for await (const chunk of process.stdin) chunks.push(chunk);
|
||||
const output = {};
|
||||
for (const { name, path: filePath } of JSON.parse(Buffer.concat(chunks).toString('utf8'))) {
|
||||
const file = await fs.openAsBlob(filePath);
|
||||
for (const entry of JSON.parse(Buffer.concat(chunks).toString('utf8'))) {
|
||||
const { name } = entry;
|
||||
if (entry.path === undefined) {
|
||||
output[name] = { classified: null, result: { selection: selectVideoCarrier(entry.tags, { limited: entry.limited || [] }) } };
|
||||
continue;
|
||||
}
|
||||
const file = await fs.openAsBlob(entry.path);
|
||||
const prefix = new Uint8Array(await file.slice(0, Math.min(32, file.size)).arrayBuffer());
|
||||
const classified = classifyPrefix(prefix);
|
||||
let result = null;
|
||||
if (classified?.kind === 'png') result = await readPngMetadataFile(file, { prefix });
|
||||
else if (classified?.media === 'image') result = await readImageHeader(file, classified.kind);
|
||||
else if (classified?.media === 'video') {
|
||||
const read = await readVideoTags(file, { prefix, kind: classified.kind });
|
||||
result = { read, selection: read.status === 'complete' ? selectVideoCarrier(read.tags, { limited: read.limited }) : null };
|
||||
}
|
||||
output[name] = { classified, result };
|
||||
}
|
||||
process.stdout.write(JSON.stringify(output));
|
||||
|
||||
@@ -0,0 +1,484 @@
|
||||
// Run with: node --test tests/js/
|
||||
// The browser video tag readers mirror what the server reads through ffprobe
|
||||
// (utils/prompt_inspector.py _video_metadata) and the carrier rule of
|
||||
// utils/video_metadata.py select_video_carrier. Each rule names its server twin;
|
||||
// tests/test_local_inspect_parity.py proves the two agree on the committed fixtures
|
||||
// and on recorded tag maps.
|
||||
import test from "node:test";
|
||||
import assert from "node:assert/strict";
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import * as v from "./helpers/video-bytes.mjs";
|
||||
|
||||
const here = path.dirname(fileURLToPath(import.meta.url));
|
||||
const FIXTURES = path.join(here, "..", "fixtures", "local-inspect");
|
||||
const RECORDED = JSON.parse(fs.readFileSync(path.join(FIXTURES, "video-tags-v1.json"), "utf8")).fixtures;
|
||||
const m = await import("../../web/js/workspace/local-video.mjs").catch((error) => {
|
||||
if (error.code !== "ERR_MODULE_NOT_FOUND") throw error;
|
||||
return {};
|
||||
});
|
||||
const { readVideoTags, selectVideoCarrier, pythonJsonDumps, pythonJsonLoads } = m;
|
||||
|
||||
const MiB = 1024 * 1024;
|
||||
const GiB = 1024 * MiB;
|
||||
const ROLE_BUDGET = 2 * MiB;
|
||||
const WORKFLOW_BUDGET = 4 * MiB;
|
||||
const TAG_READ_BYTES = 12 * MiB + 64 * 1024;
|
||||
const PER_KEY = new Set(["prompt", "workflow", "parameters", "comfyassets_prompt_pairs"]);
|
||||
const GRAPH = '{"3":{"class_type":"KSampler","inputs":{"seed":7}}}';
|
||||
const WORKFLOW = '{"nodes":[],"version":0.4}';
|
||||
|
||||
function knownTags(map) {
|
||||
return Object.fromEntries(Object.entries(map).filter(([name]) => PER_KEY.has(name.toLowerCase()) || name === "comment" || name === "description"));
|
||||
}
|
||||
|
||||
function fixture(name) {
|
||||
return new Uint8Array(fs.readFileSync(path.join(FIXTURES, name)));
|
||||
}
|
||||
|
||||
async function read(bytes, options = {}) {
|
||||
return readVideoTags(v.blobOf(bytes), options);
|
||||
}
|
||||
|
||||
function fill(byte, count) {
|
||||
return Buffer.alloc(count, byte);
|
||||
}
|
||||
|
||||
function complete(tags, extra = {}) {
|
||||
return { status: "complete", tags, limited: [], ...extra };
|
||||
}
|
||||
|
||||
test("contract: the readers, the carrier rule and the json port are exported", () => {
|
||||
assert.equal(typeof readVideoTags, "function");
|
||||
assert.equal(typeof selectVideoCarrier, "function");
|
||||
assert.equal(typeof pythonJsonDumps, "function");
|
||||
assert.equal(typeof pythonJsonLoads, "function");
|
||||
});
|
||||
|
||||
test("iso-bmff reader finds per-key tags with moov after mdat", async () => {
|
||||
for (const [name, recorded] of Object.entries(RECORDED)) {
|
||||
const blob = v.recordingBlob(fixture(name));
|
||||
const result = await readVideoTags(blob, { prefix: fixture(name).subarray(0, 32) });
|
||||
assert.equal(result.status, "complete", name);
|
||||
assert.deepEqual(result.tags, knownTags(recorded), name);
|
||||
assert.equal(result.container, name.endsWith(".mkv") || name.endsWith(".webm") ? "matroska" : "iso-bmff", name);
|
||||
assert.ok(["isom", "qt", "M4V", "matroska", "webm"].includes(result.format), `${name}: ${result.format}`);
|
||||
assert.deepEqual(result.limited, [], name);
|
||||
assert.ok(blob.ranges.length > 0, name);
|
||||
}
|
||||
// core-moov-end.mp4 behind a 3 GiB mdat: only the box headers and the tag bytes are sliced.
|
||||
const bytes = fixture("core-moov-end.mp4");
|
||||
const boxes = v.topLevelBoxes(bytes);
|
||||
const mdat = boxes.find((box) => box.type === "mdat");
|
||||
const moov = boxes.find((box) => box.type === "moov");
|
||||
const mdatSize = 3 * GiB + 16;
|
||||
const header = v.concat(v.u32(1), v.latin1("mdat"), new Uint8Array(8));
|
||||
new DataView(header.buffer).setBigUint64(8, BigInt(mdatSize));
|
||||
const moovOffset = mdat.start + mdatSize;
|
||||
const blob = v.sparseBlob(
|
||||
[
|
||||
{ offset: 0, bytes: bytes.subarray(0, mdat.start) },
|
||||
{ offset: mdat.start, bytes: header },
|
||||
{ offset: moovOffset, bytes: bytes.subarray(moov.start, moov.end) },
|
||||
],
|
||||
moovOffset + (moov.end - moov.start),
|
||||
);
|
||||
const result = await readVideoTags(blob, { prefix: bytes.subarray(0, 32) });
|
||||
assert.deepEqual(result.tags, knownTags(RECORDED["core-moov-end.mp4"]));
|
||||
const inside = blob.ranges.filter(([start]) => start >= mdat.start && start < moovOffset);
|
||||
assert.ok(inside.every(([start, end]) => start === mdat.start && end - start <= 16), JSON.stringify(inside));
|
||||
const total = blob.ranges.reduce((sum, [start, end]) => sum + (end - start), 0);
|
||||
assert.ok(total <= m.VIDEO_READ_CEILING && total < 64 * 1024, `${total}`);
|
||||
});
|
||||
|
||||
test("iso-bmff: legacy ilst atoms, moov/meta, sticky mdta handler and data types", async () => {
|
||||
// ©cmt and desc map to comment and description, as ffprobe reports them.
|
||||
const legacy = await read(v.mp4(v.legacyUdta({ comment: '{"prompt":"{}"}', description: "note" })));
|
||||
assert.deepEqual(legacy, complete({ comment: '{"prompt":"{}"}', description: "note" }, { container: "iso-bmff", format: "isom" }));
|
||||
// moov/meta (ISO style) is read like moov/udta/meta.
|
||||
const isoMeta = await read(v.mp4(v.box("meta", v.u32(0), v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH))))));
|
||||
assert.deepEqual(isoMeta.tags, { prompt: GRAPH });
|
||||
// meta without the version/flags word (QuickTime) still finds hdlr.
|
||||
const qtMeta = await read(v.mp4(v.udta(v.box("meta", v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH)))))));
|
||||
assert.deepEqual(qtMeta.tags, { prompt: GRAPH });
|
||||
// Index items need an mdta handler and a keys box; with a legacy handler they are skipped.
|
||||
const noKeys = await read(v.mp4(v.udta(v.meta(v.hdlr("mdir"), v.ilst(v.item(1, v.data(GRAPH)))))));
|
||||
assert.deepEqual(noKeys.tags, {});
|
||||
// The mdta handler is sticky across the file, as ffmpeg's found_hdlr_mdta.
|
||||
const sticky = await read(v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"])), v.meta(v.hdlr("mdir"), v.ilst(v.item(1, v.data(GRAPH)))))));
|
||||
assert.deepEqual(sticky.tags, { prompt: GRAPH });
|
||||
// As ffprobe reports them: consecutive data children join with ";", integer data
|
||||
// types print as decimals, other types are not tags, text types 0/1/4 are read, NUL
|
||||
// truncates, and the run of data children stops at the first other child.
|
||||
const mixed = await read(v.mp4(v.udta(v.meta(
|
||||
v.hdlr("mdta"),
|
||||
v.keys(["prompt", "workflow", "parameters", "comfyassets_prompt_pairs", "comment", "description"]),
|
||||
v.ilst(
|
||||
v.item(1, v.data("first"), v.data("second"), v.box("name", "x"), v.data("not read")),
|
||||
v.item(2, v.data([0, 0, 0, 7], { type: 21 }), v.data([0xff, 0xfe], { type: 21 }), v.data([0xff, 0xfe], { type: 22 })),
|
||||
v.item(3, v.data("cut\0off")),
|
||||
v.item(4, v.data("sort", { type: 4 }), v.data("mac", { type: 0 })),
|
||||
v.item(5, v.data([0, 0, 0, 0, 0, 0, 0], { type: 21 })),
|
||||
v.item(6, v.data("float", { type: 23 })),
|
||||
),
|
||||
))));
|
||||
assert.deepEqual(mixed.tags, { prompt: "first;second", workflow: "7;-2;65534", parameters: "cut", comfyassets_prompt_pairs: "sort;mac" });
|
||||
// Keys outside the mdta namespace address nothing; unknown names are skipped unread.
|
||||
const blob = v.recordingBlob(v.mp4(v.udta(v.meta(
|
||||
v.hdlr("mdta"),
|
||||
v.keys([["prompt", "udta"], "encoder", "workflow"]),
|
||||
v.ilst(v.item(1, v.data(GRAPH)), v.item(2, v.data(fill(0x41, 5000))), v.item(3, v.data(WORKFLOW))),
|
||||
))));
|
||||
const result = await readVideoTags(blob);
|
||||
assert.deepEqual(result.tags, { workflow: WORKFLOW });
|
||||
assert.ok(blob.ranges.every(([start, end]) => end - start < 5000), "the encoder value was never sliced");
|
||||
// Legacy fourccs are mapped even in mdta mode, and a repeated name joins with ";".
|
||||
const repeated = await read(v.mp4(v.udta(v.meta(
|
||||
v.hdlr("mdta"),
|
||||
v.keys(["prompt", "prompt", "comment"]),
|
||||
v.ilst(v.item(1, v.data("one")), v.item(2, v.data("two")), v.item(3, v.data("mdta")), v.legacyItem("©cmt", v.data("c"))),
|
||||
))));
|
||||
assert.deepEqual(repeated.tags, { prompt: "one;two", comment: "mdta;c" });
|
||||
// Apple's namespaced keys are reported verbatim, so they are never an envelope.
|
||||
const apple = await read(v.perKeyMp4({ "com.apple.quicktime.description": '{"prompt":"{}"}', prompt: GRAPH }));
|
||||
assert.deepEqual(apple.tags, { prompt: GRAPH });
|
||||
// An item whose first child is not a data atom, or a data atom shorter than its
|
||||
// 16-byte header, is skipped as ffmpeg skips it.
|
||||
const junk = await read(v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.box("name", "x"), v.data(GRAPH)))))));
|
||||
assert.deepEqual(junk.tags, {});
|
||||
const short = await read(v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH, { size: 12 })))))));
|
||||
assert.deepEqual(short.tags, {});
|
||||
// A box declared with size 0 runs to the end of its parent; faststart puts moov first.
|
||||
const fast = await read(v.perKeyMp4({ prompt: GRAPH }, { faststart: true, brand: "mp42" }));
|
||||
assert.deepEqual(fast, complete({ prompt: GRAPH }, { container: "iso-bmff", format: "mp42" }));
|
||||
const toEnd = await read(v.concat(v.ftyp(), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH }), v.opts({ size: 0 }))));
|
||||
assert.deepEqual(toEnd.tags, { prompt: GRAPH });
|
||||
});
|
||||
|
||||
test("matroska reads global Tags and skips clusters", async () => {
|
||||
const { ID } = v;
|
||||
// ffmpeg layout: SeekHead, Info, Tracks, Tags, Cluster, Cues; names upper-cased.
|
||||
const plain = await read(v.perKeyMkv({ PROMPT: GRAPH, WORKFLOW, ENCODER: "Lavf" }, { docType: "webm" }));
|
||||
assert.deepEqual(plain, complete({ PROMPT: GRAPH, WORKFLOW }, { container: "matroska", format: "webm" }));
|
||||
// Tags after a 3 GiB Cluster, located through the SeekHead: no slice inside the cluster.
|
||||
const tags = v.globalTags({ PROMPT: GRAPH });
|
||||
const clusterSize = 3 * GiB;
|
||||
// The Tags position (relative to the Segment payload) is a 4-byte uint either way,
|
||||
// so the SeekHead's length can be measured with a placeholder first.
|
||||
const seekLength = v.seekHead([[ID.Tags, clusterSize]]).length;
|
||||
const seekHead = v.seekHead([[ID.Tags, seekLength + 12 + clusterSize]]);
|
||||
assert.equal(seekHead.length, seekLength);
|
||||
const head = v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(seekHead.length + 12 + clusterSize + tags.length, { width: 8 }), seekHead, v.clusterHeader(clusterSize));
|
||||
const clusterStart = v.ebmlHeader().length + 12 + seekHead.length;
|
||||
const tagsOffset = head.length + clusterSize;
|
||||
const blob = v.sparseBlob([{ offset: 0, bytes: head }, { offset: tagsOffset, bytes: tags }], tagsOffset + tags.length);
|
||||
const seeked = await readVideoTags(blob, { prefix: head.subarray(0, 32) });
|
||||
assert.deepEqual(seeked.tags, { PROMPT: GRAPH });
|
||||
const inside = blob.ranges.filter(([start]) => start >= clusterStart && start < tagsOffset);
|
||||
assert.ok(inside.every(([start, end]) => start === clusterStart && end - start <= 16), JSON.stringify(inside));
|
||||
assert.ok(blob.ranges.reduce((sum, [start, end]) => sum + (end - start), 0) <= m.VIDEO_READ_CEILING);
|
||||
// ffprobe reads level-1 elements up to the first Cluster: Tags behind it are found
|
||||
// only through a SeekHead, so without one they are not read (nor sliced).
|
||||
const behind = v.recordingBlob(v.mkv([v.cluster(64), v.globalTags({ PROMPT: GRAPH }), el(ID.Cues, new Uint8Array(2))]));
|
||||
const sequential = await readVideoTags(behind);
|
||||
assert.deepEqual(sequential.tags, {});
|
||||
assert.ok(behind.ranges.every(([, end]) => end <= v.ebmlHeader().length + 12 + 12 + 32), JSON.stringify(behind.ranges));
|
||||
// Track-targeted Tags are ignored; chapter and attachment targets too.
|
||||
const targeted = await read(v.mkv([
|
||||
v.tagsEl(
|
||||
v.tag([v.simpleTag("PROMPT", "track")], { trackUid: 1 }),
|
||||
v.tag([v.simpleTag("WORKFLOW", WORKFLOW)], { targetType: 50 }),
|
||||
v.tag([v.simpleTag("PROMPT", "zero uid is global")], { trackUid: 0 }),
|
||||
),
|
||||
v.cluster(),
|
||||
]));
|
||||
assert.deepEqual(targeted.tags, { WORKFLOW, PROMPT: "zero uid is global" });
|
||||
// An unknown-size Cluster (live recordings) is the same stop; a SeekHead still works.
|
||||
const hidden = await read(v.mkv([v.cluster(8, { unknown: true }), v.globalTags({ PROMPT: GRAPH })]));
|
||||
assert.deepEqual(hidden, complete({}, { container: "matroska", format: "matroska" }));
|
||||
const before = await read(v.mkv([v.globalTags({ PROMPT: GRAPH }), v.cluster(8, { unknown: true })]));
|
||||
assert.deepEqual(before.tags, { PROMPT: GRAPH });
|
||||
const info = el(ID.Info, new Uint8Array(4));
|
||||
const clusterUnknown = v.cluster(8, { unknown: true });
|
||||
const seekToTags = v.seekHead([[ID.Tags, 0x7f]]); // a one-byte position, like the real one
|
||||
const viaSeek = await read(v.mkv([v.seekHead([[ID.Tags, seekToTags.length + info.length + clusterUnknown.length]]), info, clusterUnknown, v.globalTags({ PROMPT: GRAPH })]));
|
||||
assert.deepEqual(viaSeek.tags, { PROMPT: GRAPH });
|
||||
// An unknown-size Segment is fine (live recordings); other unknown sizes are invalid.
|
||||
const liveSegment = await read(v.mkv([v.globalTags({ PROMPT: GRAPH }), v.cluster()], { unknown: true }));
|
||||
assert.deepEqual(liveSegment.tags, { PROMPT: GRAPH });
|
||||
const unknownTags = await read(v.mkv([el(ID.Tags, v.tag([v.simpleTag("PROMPT", GRAPH)]), v.opts({ unknown: true }))]));
|
||||
assert.deepEqual(unknownTags, { status: "unreadable", reason: "invalid-container" });
|
||||
// ffmpeg's tag naming (checked against ffprobe 9.0.2): a language other than und
|
||||
// files the tag under NAME-lang only, unless TagDefault is set; nested SimpleTags
|
||||
// become PARENT/CHILD (never ours); a string-less tag deletes the key; a repeated
|
||||
// name keeps the last value; NUL ends the string; Void and CRC-32 are skipped.
|
||||
const naming = await read(v.mkv([
|
||||
v.voidEl(8),
|
||||
v.tagsEl(
|
||||
el(ID.CRC32, new Uint8Array(4)),
|
||||
v.tag([
|
||||
v.simpleTag("PROMPT", "eng only", { lang: "eng" }),
|
||||
v.simpleTag("WORKFLOW", "eng only", { lang: "eng", def: 0 }),
|
||||
v.simpleTag("workflow", "default and eng", { lang: "eng", def: 1 }),
|
||||
v.simpleTag("PARAMETERS", "und", { lang: "und", nested: [v.simpleTag("prompt", "nested")] }),
|
||||
v.simpleTag("comfyassets_prompt_pairs", "cut\0here"),
|
||||
v.simpleTag("description", undefined, { binary: [1, 2, 3] }),
|
||||
v.simpleTag("comment", "note"),
|
||||
v.simpleTag("comment", undefined),
|
||||
v.simpleTag("prompt", "first"),
|
||||
v.simpleTag("prompt", "last", { binary: [1] }),
|
||||
]),
|
||||
),
|
||||
v.cluster(),
|
||||
]));
|
||||
assert.deepEqual(naming.tags, { workflow: "default and eng", PARAMETERS: "und", comfyassets_prompt_pairs: "cut", prompt: "last" });
|
||||
// Long names are never ours and are skipped without reading their strings.
|
||||
const longName = v.recordingBlob(v.mkv([v.tagsEl(v.tag([v.simpleTag("N".repeat(300), fill(0x41, 4000)), v.simpleTag("PROMPT", GRAPH)])), v.cluster()]));
|
||||
const skipped = await readVideoTags(longName);
|
||||
assert.deepEqual(skipped.tags, { PROMPT: GRAPH });
|
||||
assert.ok(longName.ranges.every(([start, end]) => end - start < 300));
|
||||
function el(id, ...parts) {
|
||||
return v.el(id, ...parts);
|
||||
}
|
||||
});
|
||||
|
||||
test("containers fail closed", async () => {
|
||||
const { ID } = v;
|
||||
const invalid = { status: "unreadable", reason: "invalid-container" };
|
||||
const rows = [
|
||||
["box size below its header", v.concat(v.ftyp(), v.box("free", v.opts({ size: 4 })), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH })))],
|
||||
["child box past its parent", v.concat(v.ftyp(), v.mdat(), v.moov(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH), v.opts({ size: 5000 })))))))],
|
||||
["top-level box past the end of the file", v.concat(v.ftyp(), v.mdat(), v.box("moov", v.opts({ size: 4000 })))],
|
||||
["64-bit size above MAX_SAFE_INTEGER", v.concat(v.ftyp(), v.box("mdat", v.opts({ largesize: 2 ** 60 })), v.moov(v.mdtaUdta({ prompt: GRAPH })))],
|
||||
["64-bit size below its header", v.concat(v.ftyp(), v.box("mdat", v.opts({ largesize: 12 })), v.moov(v.mdtaUdta({ prompt: GRAPH })))],
|
||||
["keys count larger than the entries", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"], { count: 3 }), v.ilst(v.item(1, v.data(GRAPH))))))],
|
||||
["keys entry overrunning the box", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.box("keys", v.u32(0), v.u32(1), v.u32(90), v.latin1("mdta"), v.utf8("prompt")), v.ilst(v.item(1, v.data(GRAPH))))))],
|
||||
["ilst index 0", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(0, v.data(GRAPH))))))],
|
||||
["ilst index out of range", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(2, v.data(GRAPH))))))],
|
||||
["invalid UTF-8 in a tag value", v.perKeyMp4({ prompt: [0x7b, 0xff, 0xfe, 0x7d] })],
|
||||
["invalid UTF-8 in a key name", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys([Uint8Array.from([0xc3, 0x28])]), v.ilst(v.item(1, v.data(GRAPH))))))],
|
||||
["no moov box", v.concat(v.ftyp(), v.mdat(), v.free(32))],
|
||||
["not a container at all", v.utf8("#EXTM3U\n#EXT-X-VERSION:3\n".padEnd(64, "x"))],
|
||||
["bad VINT: zero ID byte", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(20, { width: 8 }), [0x00, 0x81, 0x00], new Uint8Array(17))],
|
||||
["bad VINT: zero size byte", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(20, { width: 8 }), v.idBytes(ID.Info), [0x00], new Uint8Array(17))],
|
||||
["element past its parent", v.mkv([v.el(ID.Tags, v.tag([v.simpleTag("PROMPT", GRAPH)]), v.opts({ size: 5000 }))])],
|
||||
["element past the end of the file", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(5000, { width: 8 }), v.globalTags({ PROMPT: GRAPH }))],
|
||||
["unknown DocType", v.perKeyMkv({ PROMPT: GRAPH }, { docType: "avi" })],
|
||||
["EBML header without a DocType", v.concat(v.el(ID.EBML, v.el(ID.EBMLVersion, [1])), v.segment([v.globalTags({ PROMPT: GRAPH })]))],
|
||||
["invalid UTF-8 in a TagString", v.mkv([v.tagsEl(v.tag([v.simpleTag("PROMPT", [0xff, 0xfe])]))])],
|
||||
["invalid UTF-8 in a TagName", v.mkv([v.tagsEl(v.tag([v.simpleTag([0xff], GRAPH)]))])],
|
||||
["Matroska bytes after a stray size", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(3, { width: 8 }), [0xec, 0x82, 0x00])],
|
||||
];
|
||||
for (const [name, bytes] of rows) {
|
||||
assert.deepEqual(await read(bytes), invalid, name);
|
||||
}
|
||||
// Trailing bytes shorter than a box header are ignored, as ffmpeg does.
|
||||
assert.deepEqual((await read(v.concat(v.perKeyMp4({ prompt: GRAPH }), [1, 2, 3]))).tags, { prompt: GRAPH });
|
||||
// 10,001 boxes stop the read.
|
||||
const many = v.concat(v.ftyp(), ...Array.from({ length: 10001 }, () => v.free()), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH })));
|
||||
assert.deepEqual(await read(many), { status: "limited", reason: "chunk-limit" });
|
||||
const manyElements = v.mkv([...Array.from({ length: 10001 }, () => v.voidEl(1)), v.globalTags({ PROMPT: GRAPH })]);
|
||||
assert.deepEqual(await read(manyElements), { status: "limited", reason: "chunk-limit" });
|
||||
// Exactly 10,000 elements still read (ftyp, mdat, moov, udta, meta, hdlr, keys, its entry, ilst, the item and its data atom).
|
||||
const exact = v.concat(v.ftyp(), ...Array.from({ length: 10000 - 11 }, () => v.free()), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH })));
|
||||
assert.deepEqual((await read(exact)).tags, { prompt: GRAPH });
|
||||
// A vanished file is a read failure, not a container error.
|
||||
assert.deepEqual(await readVideoTags(v.failingBlob(4096), { prefix: v.ftyp() }), { status: "unreadable", reason: "read-failure" });
|
||||
assert.deepEqual(await readVideoTags({}), { status: "unreadable", reason: "read-failure" });
|
||||
});
|
||||
|
||||
test("budgets: roles share 2 MiB, an oversized workflow is left unread, duplicates and the total are bounded", async () => {
|
||||
const limited = { status: "limited", reason: "metadata-limit" };
|
||||
// Role tags over 2 MiB together stop the read (the server fails the whole read).
|
||||
assert.deepEqual(await read(v.perKeyMp4({ prompt: fill(0x41, ROLE_BUDGET + 1) })), limited);
|
||||
assert.deepEqual(await read(v.perKeyMkv({ PROMPT: fill(0x41, ROLE_BUDGET - 10), PARAMETERS: fill(0x42, 11) })), limited);
|
||||
const atRole = await read(v.perKeyMp4({ prompt: fill(0x41, ROLE_BUDGET - 10), comfyassets_prompt_pairs: fill(0x42, 10) }));
|
||||
assert.equal(atRole.status, "complete");
|
||||
assert.equal(atRole.tags.prompt.length, ROLE_BUDGET - 10);
|
||||
// A per-key workflow over 4 MiB is not sliced; the result says the workflow is limited.
|
||||
const big = v.recordingBlob(v.perKeyMp4({ prompt: GRAPH, workflow: fill(0x7b, WORKFLOW_BUDGET + 1) }));
|
||||
const over = await readVideoTags(big);
|
||||
assert.deepEqual(over, { status: "complete", container: "iso-bmff", format: "isom", tags: { prompt: GRAPH }, limited: ["workflow"] });
|
||||
assert.ok(big.ranges.every(([start, end]) => end - start < WORKFLOW_BUDGET), "the oversized workflow was never sliced");
|
||||
const bigMkv = await read(v.perKeyMkv({ WORKFLOW: fill(0x7b, WORKFLOW_BUDGET + 1), PROMPT: GRAPH }));
|
||||
assert.deepEqual([bigMkv.limited, Object.keys(bigMkv.tags)], [["workflow"], ["PROMPT"]]);
|
||||
const atWorkflow = await read(v.perKeyMp4({ workflow: fill(0x7b, WORKFLOW_BUDGET) }));
|
||||
assert.deepEqual([atWorkflow.limited, atWorkflow.tags.workflow.length], [[], WORKFLOW_BUDGET]);
|
||||
// Fold-duplicates draw from the duplicate budgets (2x each class), then stop.
|
||||
const twice = await read(v.perKeyMkv({ PROMPT: fill(0x41, ROLE_BUDGET), prompt: fill(0x41, ROLE_BUDGET) }));
|
||||
assert.deepEqual(Object.keys(twice.tags).sort(), ["PROMPT", "prompt"]);
|
||||
const thrice = await read(v.perKeyMkv({ PROMPT: fill(0x41, ROLE_BUDGET), prompt: fill(0x41, ROLE_BUDGET), Prompt: fill(0x41, ROLE_BUDGET + 1) }));
|
||||
assert.deepEqual(thrice, limited);
|
||||
const workflowTwice = await read(v.perKeyMkv({ WORKFLOW: fill(0x7b, WORKFLOW_BUDGET), workflow: fill(0x7b, WORKFLOW_BUDGET) }));
|
||||
assert.equal(workflowTwice.status, "complete");
|
||||
// Envelopes count against the total only; the total is the server's probe bound.
|
||||
const envelope = await read(v.mp4(v.legacyUdta({ comment: fill(0x41, 5 * MiB) })));
|
||||
assert.equal(envelope.tags.comment.length, 5 * MiB);
|
||||
const tooMuch = await read(v.mp4(v.legacyUdta({ comment: fill(0x41, TAG_READ_BYTES - 10), description: fill(0x42, 11) })));
|
||||
assert.deepEqual(tooMuch, limited);
|
||||
assert.equal(m.VIDEO_TAG_READ_BYTES, TAG_READ_BYTES);
|
||||
});
|
||||
|
||||
test("avi and asf are recognised but not read", async () => {
|
||||
const avi = v.concat(v.latin1("RIFF"), v.u32(1000), v.latin1("AVI LIST"), new Uint8Array(16));
|
||||
const asf = v.concat([0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c], new Uint8Array(16));
|
||||
for (const bytes of [avi, asf]) {
|
||||
const blob = v.recordingBlob(v.concat(bytes, new Uint8Array(4096)));
|
||||
assert.deepEqual(await readVideoTags(blob, { prefix: bytes }), { status: "unsupported", reason: "video-container-unsupported" });
|
||||
assert.deepEqual(blob.ranges, []);
|
||||
}
|
||||
// The classifier's kind can be handed in instead of a prefix.
|
||||
const blob = v.recordingBlob(new Uint8Array(64));
|
||||
assert.deepEqual(await readVideoTags(blob, { kind: "avi" }), { status: "unsupported", reason: "video-container-unsupported" });
|
||||
assert.deepEqual(blob.ranges, []);
|
||||
// Without a prefix the reader slices the first 32 bytes itself.
|
||||
const mp4 = v.recordingBlob(v.perKeyMp4({ prompt: GRAPH }));
|
||||
assert.deepEqual((await readVideoTags(mp4)).tags, { prompt: GRAPH });
|
||||
assert.deepEqual(mp4.ranges[0], [0, 32]);
|
||||
});
|
||||
|
||||
test("abort during a moov walk settles once", async () => {
|
||||
const unhandled = [];
|
||||
const onUnhandled = (reason) => unhandled.push(reason);
|
||||
process.on("unhandledRejection", onUnhandled);
|
||||
try {
|
||||
const bytes = v.concat(v.ftyp(), ...Array.from({ length: 50 }, () => v.free()), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH, workflow: WORKFLOW })));
|
||||
const controller = new AbortController();
|
||||
let slices = 0;
|
||||
const blob = {
|
||||
size: bytes.length,
|
||||
slice(start, end) {
|
||||
slices += 1;
|
||||
if (slices === 3) controller.abort();
|
||||
return { arrayBuffer: () => new Promise((resolve) => setTimeout(() => resolve(bytes.slice(start, end).buffer), 2)) };
|
||||
},
|
||||
};
|
||||
await assert.rejects(readVideoTags(blob, { signal: controller.signal }), { name: "AbortError" });
|
||||
const already = new AbortController();
|
||||
already.abort();
|
||||
await assert.rejects(readVideoTags(v.blobOf(bytes), { signal: already.signal }), { name: "AbortError" });
|
||||
const slow = v.slowBlob(bytes, 1);
|
||||
const during = new AbortController();
|
||||
const pending = readVideoTags(slow, { signal: during.signal, yieldControl: async () => during.abort() });
|
||||
await assert.rejects(pending, { name: "AbortError" });
|
||||
await new Promise((resolve) => setTimeout(resolve, 10));
|
||||
assert.deepEqual(unhandled, []);
|
||||
} finally {
|
||||
process.off("unhandledRejection", onUnhandled);
|
||||
}
|
||||
});
|
||||
|
||||
test("yieldControl is awaited while walking many boxes", async () => {
|
||||
let yields = 0;
|
||||
const many = v.concat(v.ftyp(), ...Array.from({ length: 3000 }, () => v.free()), v.mdat(), v.moov(v.mdtaUdta({ prompt: GRAPH })));
|
||||
const result = await readVideoTags(v.slowBlob(many, 0), { yieldControl: async () => { yields += 1; } });
|
||||
assert.equal(result.status, "complete");
|
||||
assert.ok(yields > 0, "the reader handed control back");
|
||||
});
|
||||
|
||||
test("carrier selection: per-key tags, then the comment envelope, then description, never merged", () => {
|
||||
const ENVELOPE_WORKFLOW = '{"last_node_id": 9, "nodes": [{"id": 3, "type": "KSampler", "widgets_values": [7, "fixed", 20, 8.0, "euler", "normal", 1.0]}, {"id": 6, "type": "CLIPTextEncode", "widgets_values": ["a red fox in snow, detailed fur"]}, {"id": 7, "type": "CLIPTextEncode", "widgets_values": ["blurry, lowres"]}], "links": [], "version": 0.4}';
|
||||
const original = (keys) => Object.fromEntries(keys.map((key) => [key, "original-text"]));
|
||||
const text = (tags, keys) => Object.fromEntries(keys.map((key) => [key, tags[key] ?? tags[key.toUpperCase()]]));
|
||||
// The recorded fixtures.
|
||||
for (const name of ["core-faststart.mp4", "core-moov-end.mp4", "core.mov", "core.m4v", "core.mkv", "core-tags.webm"]) {
|
||||
const tags = RECORDED[name];
|
||||
const keys = Object.keys(tags).map((key) => key.toLowerCase()).filter((key) => PER_KEY.has(key));
|
||||
assert.deepEqual(selectVideoCarrier(tags), { status: "complete", records: text(tags, keys), record_formats: original(keys), carrierText: null, limited: [] }, name);
|
||||
}
|
||||
const legacy = RECORDED["legacy-comment.mp4"];
|
||||
const comment = pythonJsonLoads(legacy.comment);
|
||||
assert.deepEqual(selectVideoCarrier(legacy), {
|
||||
status: "complete",
|
||||
records: { prompt: comment.get("prompt"), workflow: ENVELOPE_WORKFLOW },
|
||||
record_formats: { prompt: "original-text", workflow: "reserialized-json" },
|
||||
carrierText: legacy.comment,
|
||||
limited: [],
|
||||
});
|
||||
assert.ok(comment.get("prompt").startsWith('{"3":{"class_type":"KSampler"'));
|
||||
// Matroska upper-cases COMMENT, and envelope names are exact-case (ruling M1): nothing is read.
|
||||
assert.deepEqual(selectVideoCarrier(RECORDED["legacy-comment.webm"]), { status: "complete", records: {}, record_formats: {}, carrierText: null, limited: [] });
|
||||
// Synthetic maps.
|
||||
const envelope = JSON.stringify({ prompt: '{"9":{}}', workflow: { v: 1 } });
|
||||
const cases = [
|
||||
["per-key wins over comment", { prompt: GRAPH, workflow: WORKFLOW, comment: envelope }, { records: { prompt: GRAPH, workflow: WORKFLOW }, record_formats: original(["prompt", "workflow"]), carrierText: null }],
|
||||
["Matroska names fold", { PROMPT: GRAPH, Workflow: WORKFLOW, ENCODER: "Lavf" }, { records: { prompt: GRAPH, workflow: WORKFLOW }, record_formats: original(["prompt", "workflow"]), carrierText: null }],
|
||||
["identical fold-duplicates are one record", { PROMPT: GRAPH, prompt: GRAPH }, { records: { prompt: GRAPH }, record_formats: original(["prompt"]), carrierText: null }],
|
||||
["differing fold-duplicates fail", { PROMPT: GRAPH, prompt: '{"9":{}}' }, { status: "unreadable", reason: "duplicate-metadata" }],
|
||||
["differing workflow duplicates fail even with a prompt", { Workflow: WORKFLOW, WORKFLOW: "{}", prompt: GRAPH }, { status: "unreadable", reason: "duplicate-metadata" }],
|
||||
["envelope names never fold", { comment: envelope, COMMENT: "a free-text note" }, { records: { prompt: '{"9":{}}', workflow: '{"v": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: envelope }],
|
||||
["non-JSON comment falls through to description", { comment: "not json", description: envelope }, { records: { prompt: '{"9":{}}', workflow: '{"v": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: envelope }],
|
||||
["a JSON array comment is not an envelope", { comment: '["prompt"]', description: '{"workflow": {}}' }, { records: { workflow: "{}" }, record_formats: { workflow: "reserialized-json" }, carrierText: '{"workflow": {}}' }],
|
||||
["a role envelope beats a workflow-only per-key carrier", { workflow: WORKFLOW, comment: envelope }, { records: { prompt: '{"9":{}}', workflow: '{"v": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: envelope }],
|
||||
["role description beats workflow-only comment", { comment: '{"workflow": {"a": 1}}', description: '{"parameters": "cat"}' }, { records: { parameters: "cat" }, record_formats: { parameters: "original-text" }, carrierText: '{"parameters": "cat"}' }],
|
||||
["workflow-only carriers: the first wins, never merged", { comment: '{"workflow": {"a": 1}}', description: '{"workflow": {"b": 2}}' }, { records: { workflow: '{"a": 1}' }, record_formats: { workflow: "reserialized-json" }, carrierText: '{"workflow": {"a": 1}}' }],
|
||||
["description only", { description: envelope }, { records: { prompt: '{"9":{}}', workflow: '{"v": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: envelope }],
|
||||
["no known key", { encoder: "Lavf", major_brand: "isom" }, { records: {}, record_formats: {}, carrierText: null }],
|
||||
["envelope with only unknown keys is no carrier", { comment: '{"title": "x"}' }, { records: {}, record_formats: {}, carrierText: null }],
|
||||
["null role marker is refused", { comment: '{"prompt": null, "workflow": {}}' }, { status: "unreadable", reason: "invalid-metadata" }],
|
||||
["numeric role is refused", { description: '{"parameters": 5}' }, { status: "unreadable", reason: "invalid-metadata" }],
|
||||
["a non-text non-object workflow is invalid but the roles stay", { comment: '{"prompt": "{}", "workflow": null}' }, { records: { prompt: "{}" }, record_formats: { prompt: "original-text", workflow: "invalid" }, carrierText: '{"prompt": "{}", "workflow": null}' }],
|
||||
["an object prompt is reserialized", { comment: '{"prompt": {"1": {"class_type": "A"}}}' }, { records: { prompt: '{"1": {"class_type": "A"}}' }, record_formats: { prompt: "reserialized-json" }, carrierText: '{"prompt": {"1": {"class_type": "A"}}}' }],
|
||||
["a non-text per-key role marker is refused", { prompt: 5, workflow: WORKFLOW }, { status: "unreadable", reason: "invalid-metadata" }],
|
||||
["a non-text per-key workflow is invalid", { prompt: GRAPH, workflow: 5 }, { records: { prompt: GRAPH }, record_formats: { prompt: "original-text", workflow: "invalid" }, carrierText: null }],
|
||||
["a workflow-only envelope with an invalid workflow is still the carrier", { comment: '{"workflow": 5}' }, { records: {}, record_formats: { workflow: "invalid" }, carrierText: '{"workflow": 5}' }],
|
||||
["not a map", null, { records: {}, record_formats: {}, carrierText: null }],
|
||||
];
|
||||
for (const [name, tags, expected] of cases) {
|
||||
const want = expected.status ? expected : { status: "complete", limited: [], ...expected };
|
||||
assert.deepEqual(selectVideoCarrier(tags), want, name);
|
||||
}
|
||||
// The server's budget loop: a 4 MiB workflow is limited, roles over 2 MiB fail, exact sizes pass.
|
||||
const bigWorkflow = "{" + "0".repeat(WORKFLOW_BUDGET);
|
||||
assert.deepEqual(selectVideoCarrier({ prompt: GRAPH, workflow: bigWorkflow }), { status: "complete", records: { prompt: GRAPH }, record_formats: { prompt: "original-text", workflow: "limited" }, carrierText: null, limited: ["workflow"] });
|
||||
assert.deepEqual(selectVideoCarrier({ workflow: bigWorkflow.slice(1) }).records, { workflow: bigWorkflow.slice(1) });
|
||||
assert.deepEqual(selectVideoCarrier({ comment: JSON.stringify({ prompt: "x", workflow: { k: "0".repeat(WORKFLOW_BUDGET) } }) }).record_formats, { prompt: "original-text", workflow: "limited" });
|
||||
assert.deepEqual(selectVideoCarrier({ prompt: "a".repeat(ROLE_BUDGET), parameters: "b" }), { status: "limited", reason: "metadata-limit" });
|
||||
assert.deepEqual(selectVideoCarrier({ prompt: "é".repeat(ROLE_BUDGET / 2) }).records.prompt.length, ROLE_BUDGET / 2);
|
||||
assert.deepEqual(selectVideoCarrier({ prompt: "é".repeat(ROLE_BUDGET / 2) + "a" }), { status: "limited", reason: "metadata-limit" });
|
||||
assert.deepEqual(selectVideoCarrier({ prompt: "\ud800".repeat(ROLE_BUDGET / 3) + "aaa" }), { status: "limited", reason: "metadata-limit" });
|
||||
assert.deepEqual(selectVideoCarrier({ prompt: "\u{1f600}".repeat(ROLE_BUDGET / 4) }).records.prompt.length, ROLE_BUDGET / 2);
|
||||
assert.deepEqual(selectVideoCarrier({ prompt: "\u{1f600}".repeat(ROLE_BUDGET / 4) + "a" }), { status: "limited", reason: "metadata-limit" });
|
||||
// The reader's limited workflow (the server's role-only re-probe): kept with the per-key
|
||||
// carrier or no carrier, cleared when an envelope wins.
|
||||
assert.deepEqual(selectVideoCarrier({ prompt: GRAPH }, { limited: ["workflow"] }).record_formats, { prompt: "original-text", workflow: "limited" });
|
||||
assert.deepEqual(selectVideoCarrier({ encoder: "x" }, { limited: ["workflow"] }), { status: "complete", records: {}, record_formats: { workflow: "limited" }, carrierText: null, limited: ["workflow"] });
|
||||
assert.deepEqual(selectVideoCarrier({ comment: envelope }, { limited: ["workflow"] }).limited, []);
|
||||
assert.deepEqual(selectVideoCarrier({ comment: envelope }, { limited: ["workflow"] }).record_formats, { prompt: "original-text", workflow: "reserialized-json" });
|
||||
});
|
||||
|
||||
test("pythonJsonLoads and pythonJsonDumps spell values as Python's json module does", () => {
|
||||
const rows = [
|
||||
["1.0", "1.0"], ["8.0", "8.0"], ["0.1", "0.1"], ["1e16", "1e+16"], ["1e15", "1000000000000000.0"],
|
||||
["123456789012345678", "123456789012345678"], ["0.0001", "0.0001"], ["0.00001", "1e-05"], ["1e-7", "1e-07"],
|
||||
["1.5e300", "1.5e+300"], ["1e400", "Infinity"], ["-0", "0"], ["-0.0", "-0.0"], ["1E2", "100.0"], ["2.5", "2.5"],
|
||||
["100.0", "100.0"], ["1e21", "1e+21"], ["123456.789", "123456.789"], ["0.000123", "0.000123"], ["-1e-05", "-1e-05"],
|
||||
["9007199254740993", "9007199254740993"], ["1.7976931348623157e308", "1.7976931348623157e+308"], ["5e-324", "5e-324"],
|
||||
["NaN", "NaN"], ["-Infinity", "-Infinity"], ["Infinity", "Infinity"], ["[1.0e+2]", "[100.0]"], ["-1e400", "-Infinity"],
|
||||
["[]", "[]"], ["{}", "{}"], ["[true, false, null]", "[true, false, null]"], ['{"k":[1,{"z":1}]}', '{"k": [1, {"z": 1}]}'],
|
||||
[' \t\n\r{"a" : 1 } ', '{"a": 1}'],
|
||||
['{"a":"é\\u0001\u007f\\"\\\\\\n\\t\\b\\f\\r\\/ 😀 \\ud800"}', '{"a": "\\u00e9\\u0001\\u007f\\"\\\\\\n\\t\\b\\f\\r/ \\ud83d\\ude00 \\ud800"}'],
|
||||
['{"a":"\\ud83d\\ude00"}', '{"a": "\\ud83d\\ude00"}'],
|
||||
['{"10":1,"3":2,"b":3,"a":4}', '{"10": 1, "3": 2, "b": 3, "a": 4}'],
|
||||
['{"a":1,"b":2,"a":3}', '{"a": 3, "b": 2}'],
|
||||
];
|
||||
for (const [input, output] of rows) {
|
||||
assert.equal(pythonJsonDumps(pythonJsonLoads(input)), output, input);
|
||||
}
|
||||
const bad = ['{"a":1} x', '{"a":"\u0001"}', '{"a":"\\x"}', "01", "1.", ".5", "+1", "[1,]", '{"a":1,}', "{a:1}", "'a'", "", " ", "[", "nul", "-", "1e", '"\\u12"', "[1 2]", "tru", "Infinit"];
|
||||
for (const input of bad) {
|
||||
assert.throws(() => pythonJsonLoads(input), { name: "PythonJsonError" }, JSON.stringify(input));
|
||||
}
|
||||
assert.throws(() => pythonJsonLoads("[".repeat(2000) + "]".repeat(2000)), { name: "PythonJsonError", recursion: true });
|
||||
assert.throws(() => pythonJsonDumps(undefined), { name: "PythonJsonError" });
|
||||
assert.throws(() => pythonJsonLoads(5), { name: "PythonJsonError" });
|
||||
assert.equal(pythonJsonDumps(pythonJsonLoads("[".repeat(500) + "]".repeat(500))), "[".repeat(500) + "]".repeat(500));
|
||||
const parsed = pythonJsonLoads('{"s":"x","n":1,"f":1.5,"b":true,"z":null,"l":[1]}');
|
||||
assert.ok(parsed instanceof Map);
|
||||
assert.deepEqual([...parsed.keys()], ["s", "n", "f", "b", "z", "l"]);
|
||||
assert.equal(parsed.get("s"), "x");
|
||||
assert.equal(parsed.get("b"), true);
|
||||
assert.equal(parsed.get("z"), null);
|
||||
assert.ok(Array.isArray(parsed.get("l")));
|
||||
});
|
||||
@@ -1,4 +1,4 @@
|
||||
"""The browser's local PNG reader answers exactly as the server's on the same file.
|
||||
"""The browser's local readers answer exactly as the server's on the same file.
|
||||
|
||||
``web/js/png-metadata.js`` ``readPngMetadataFile`` ports the bounded PNG reader in
|
||||
``utils/prompt_inspector.py``. This test hands the inspection parity corpus (the
|
||||
@@ -7,6 +7,15 @@ Latin-1 and batch-duplicate PNGs, and the committed JPEG/WebP header fixtures to
|
||||
Node oracle, and asserts per file that records, record formats, dimensions and
|
||||
readability equal ``read_inspection_file``. The oracle reads each file through
|
||||
``fs.openAsBlob`` slices, as the page reads a chosen ``File``.
|
||||
|
||||
``web/js/workspace/local-video.mjs`` reads the container tags the server asks
|
||||
ffprobe for and ports ``utils/video_metadata.select_video_carrier`` with the
|
||||
record budgets of ``read_inspection_file``. The video tests hand the oracle the
|
||||
committed container fixtures (``tests/fixtures/local-inspect/*.{mp4,mov,m4v,mkv,
|
||||
webm}`` with their recorded ffprobe tag maps in ``video-tags-v1.json``), recorded
|
||||
and synthetic tag maps, and multi-megabyte per-key containers built at test time,
|
||||
and assert that the local carrier equals the server's, which selects from the
|
||||
same tag map through a mocked probe, or from the same file through real ffprobe.
|
||||
"""
|
||||
|
||||
import io
|
||||
@@ -19,6 +28,7 @@ import tempfile
|
||||
import unittest
|
||||
import zlib
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from PIL import Image
|
||||
from PIL.PngImagePlugin import PngInfo
|
||||
@@ -33,16 +43,155 @@ from utils.prompt_inspector import (
|
||||
try:
|
||||
from tests import inspection_parity as parity
|
||||
from tests.open_handles import assert_closed
|
||||
from tests.test_utils_video_metadata import HAVE_FFMPEG, write_video
|
||||
except ImportError: # pragma: no cover - discover -s tests puts tests/ first
|
||||
import inspection_parity as parity
|
||||
from open_handles import assert_closed
|
||||
from test_utils_video_metadata import HAVE_FFMPEG, write_video
|
||||
|
||||
HERE = Path(__file__).resolve().parent
|
||||
ORACLE = HERE / "js" / "support" / "local-inspect-oracle.mjs"
|
||||
HEADER_FIXTURES = HERE / "fixtures" / "local-inspect"
|
||||
VIDEO_FIXTURES = HEADER_FIXTURES
|
||||
RECORDED_TAGS = json.loads(
|
||||
(VIDEO_FIXTURES / "video-tags-v1.json").read_text(encoding="utf-8")
|
||||
)["fixtures"]
|
||||
NODE_TIMEOUT_SECONDS = 180
|
||||
PNG_CLASS = {"media": "image", "kind": "png"}
|
||||
_FOOTER = "Steps: 20, Sampler: Euler, Batch index: {}, Seed: 7"
|
||||
MiB = 1024 * 1024
|
||||
GRAPH = '{"3":{"class_type":"KSampler","inputs":{"seed":7}}}'
|
||||
WORKFLOW = '{"nodes":[],"version":0.4}'
|
||||
ENVELOPE = json.dumps({"prompt": '{"9":{}}', "workflow": {"v": 1}})
|
||||
VIDEO_CONTAINERS = {
|
||||
".mp4": "iso-bmff",
|
||||
".mov": "iso-bmff",
|
||||
".m4v": "iso-bmff",
|
||||
".mkv": "matroska",
|
||||
".webm": "matroska",
|
||||
}
|
||||
PER_KEY_NAMES = {"prompt", "workflow", "parameters", "comfyassets_prompt_pairs"}
|
||||
|
||||
# Tag maps as ffprobe would report them, covering every branch of the carrier rule:
|
||||
# (name, tags, records the reader left out). The server answer is computed, not
|
||||
# written down, so a change in either side fails here.
|
||||
TAG_MAP_VECTORS = [
|
||||
(
|
||||
"per_key_wins_over_comment",
|
||||
{"prompt": GRAPH, "workflow": WORKFLOW, "comment": ENVELOPE},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"matroska_names_fold",
|
||||
{"PROMPT": GRAPH, "Workflow": WORKFLOW, "ENCODER": "Lavf"},
|
||||
[],
|
||||
),
|
||||
("identical_fold_duplicates", {"PROMPT": GRAPH, "prompt": GRAPH}, []),
|
||||
("differing_fold_duplicates_fail", {"PROMPT": GRAPH, "prompt": '{"9":{}}'}, []),
|
||||
(
|
||||
"differing_workflow_duplicates_fail",
|
||||
{"Workflow": WORKFLOW, "WORKFLOW": "{}", "prompt": GRAPH},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"envelope_names_never_fold",
|
||||
{"comment": ENVELOPE, "COMMENT": "a free-text note"},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"non_json_comment_falls_to_description",
|
||||
{"comment": "not json", "description": ENVELOPE},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"array_comment_is_not_an_envelope",
|
||||
{"comment": '["prompt"]', "description": '{"workflow": {}}'},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"role_envelope_beats_workflow_only_per_key",
|
||||
{"workflow": WORKFLOW, "comment": ENVELOPE},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"role_description_beats_workflow_comment",
|
||||
{"comment": '{"workflow": {"a": 1}}', "description": '{"parameters": "cat"}'},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"workflow_only_carriers_never_merge",
|
||||
{"comment": '{"workflow": {"a": 1}}', "description": '{"workflow": {"b": 2}}'},
|
||||
[],
|
||||
),
|
||||
("description_only", {"description": ENVELOPE}, []),
|
||||
("no_known_key", {"encoder": "Lavf", "major_brand": "isom"}, []),
|
||||
("envelope_without_known_keys", {"comment": '{"title": "x"}'}, []),
|
||||
("null_role_marker_refused", {"comment": '{"prompt": null, "workflow": {}}'}, []),
|
||||
("numeric_role_refused", {"description": '{"parameters": 5}'}, []),
|
||||
(
|
||||
"invalid_envelope_workflow_keeps_roles",
|
||||
{"comment": '{"prompt": "{}", "workflow": null}'},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"object_prompt_reserialized",
|
||||
{
|
||||
"comment": '{"prompt": {"1": {"class_type": "A", "f": 8.0, "n": 1e16, '
|
||||
'"u": "é😀"}}}'
|
||||
},
|
||||
[],
|
||||
),
|
||||
("workflow_only_invalid_envelope", {"comment": '{"workflow": 5}'}, []),
|
||||
("non_text_per_key_role_refused", {"prompt": 5, "workflow": WORKFLOW}, []),
|
||||
("non_text_per_key_workflow_invalid", {"prompt": GRAPH, "workflow": 5}, []),
|
||||
(
|
||||
"python_number_spellings",
|
||||
{
|
||||
"description": '{"workflow": {"a": [1.0, 0.1, 1E2, 1e-7, -0.0, -0, '
|
||||
"123456789012345678, 1e400, NaN, 0.00001, 1e15], "
|
||||
'"b": "\\u0001\\u007f/"}}'
|
||||
},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"integer_like_keys_keep_order",
|
||||
{"description": '{"workflow": {"10": 1, "3": 2, "b": 3, "a": 4, "3": 5}}'},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"workflow_over_budget_limited",
|
||||
{"prompt": GRAPH, "workflow": "{" + "0" * WORKFLOW_METADATA_BYTES},
|
||||
[],
|
||||
),
|
||||
("workflow_at_budget_kept", {"workflow": "0" * WORKFLOW_METADATA_BYTES}, []),
|
||||
(
|
||||
"roles_over_budget_fail",
|
||||
{"prompt": "a" * ROLE_METADATA_BYTES, "parameters": "b"},
|
||||
[],
|
||||
),
|
||||
("roles_at_budget_kept", {"prompt": "é" * (ROLE_METADATA_BYTES // 2)}, []),
|
||||
(
|
||||
"lone_surrogates_count_three_bytes",
|
||||
{"prompt": "\ud800" * (ROLE_METADATA_BYTES // 3) + "aaa"},
|
||||
[],
|
||||
),
|
||||
(
|
||||
"envelope_workflow_over_budget_limited",
|
||||
{
|
||||
"comment": json.dumps(
|
||||
{"prompt": "x", "workflow": {"k": "0" * WORKFLOW_METADATA_BYTES}}
|
||||
)
|
||||
},
|
||||
[],
|
||||
),
|
||||
("reader_limited_workflow_with_per_key", {"prompt": GRAPH}, ["workflow"]),
|
||||
("reader_limited_workflow_without_carrier", {"encoder": "x"}, ["workflow"]),
|
||||
(
|
||||
"reader_limited_workflow_cleared_by_envelope",
|
||||
{"comment": ENVELOPE},
|
||||
["workflow"],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
def _raw_chunk(kind, data):
|
||||
@@ -175,6 +324,156 @@ def _classes(results):
|
||||
return {name: entry["classified"] for name, entry in results.items()}
|
||||
|
||||
|
||||
def _run_oracle(testcase, entries):
|
||||
node = shutil.which("node")
|
||||
if node is None: # pragma: no cover - CI runners and dev hosts ship Node
|
||||
if os.environ.get("CI"):
|
||||
testcase.fail("Node.js is required in CI for the local reader parity test")
|
||||
testcase.skipTest("Node.js is not installed")
|
||||
# Fixed local Node harness, no shell.
|
||||
completed = subprocess.run( # nosec B603
|
||||
[node, str(ORACLE)],
|
||||
input=json.dumps(entries).encode("utf-8"),
|
||||
capture_output=True,
|
||||
timeout=NODE_TIMEOUT_SECONDS,
|
||||
check=False,
|
||||
)
|
||||
testcase.assertEqual(completed.returncode, 0, completed.stderr.decode("utf-8"))
|
||||
return json.loads(completed.stdout)
|
||||
|
||||
|
||||
def _known_tags(tags):
|
||||
"""The tags the local reader keeps: per-key names folded, envelope names exact."""
|
||||
return {
|
||||
key: value
|
||||
for key, value in tags.items()
|
||||
if key.lower() in PER_KEY_NAMES or key in ("comment", "description")
|
||||
}
|
||||
|
||||
|
||||
def _fake_probe(tags, drop_workflow=False):
|
||||
"""``_probe_video_tags`` answering from a tag map as ffprobe would.
|
||||
|
||||
ffprobe's ``format_tags=`` filter is case-insensitive and the stdout bound is
|
||||
the only reason a probe answers ``None``; ``drop_workflow`` makes the first
|
||||
probe (the one asking for ``workflow``) overflow, as a huge per-key workflow
|
||||
tag does, so the server takes its role-only re-probe.
|
||||
"""
|
||||
|
||||
def probe(binary, path, keys, limit):
|
||||
if drop_workflow and "workflow" in keys:
|
||||
return None
|
||||
selected = {key: value for key, value in tags.items() if key.lower() in keys}
|
||||
raw = json.dumps({"format": {"tags": selected}}, ensure_ascii=False).encode(
|
||||
"utf-8", "surrogatepass"
|
||||
)
|
||||
return None if len(raw) > limit else raw
|
||||
|
||||
return probe
|
||||
|
||||
|
||||
def _server_selection(tags, limited=()):
|
||||
"""The server's carrier for a tag map, in the oracle's shape; None when it fails."""
|
||||
with (
|
||||
patch("utils.prompt_inspector.ffprobe_path", return_value="/tool/ffprobe"),
|
||||
patch(
|
||||
"utils.prompt_inspector._probe_video_tags",
|
||||
side_effect=_fake_probe(tags, drop_workflow="workflow" in limited),
|
||||
),
|
||||
):
|
||||
try:
|
||||
read = read_inspection_file(Path("vector.mp4"))
|
||||
except MetadataReadError:
|
||||
return None
|
||||
return _read_shape(read)
|
||||
|
||||
|
||||
def _read_shape(read):
|
||||
return {
|
||||
"records": read.metadata,
|
||||
"record_formats": read.record_formats,
|
||||
"carrierText": read.video_carrier_text,
|
||||
"limited": sorted(read.limited_records),
|
||||
}
|
||||
|
||||
|
||||
def _local_shape(selection):
|
||||
if selection["status"] != "complete":
|
||||
return None
|
||||
return {
|
||||
"records": selection["records"],
|
||||
"record_formats": selection["record_formats"],
|
||||
"carrierText": selection["carrierText"],
|
||||
"limited": sorted(selection["limited"]),
|
||||
}
|
||||
|
||||
|
||||
def _box(kind, *parts):
|
||||
payload = b"".join(parts)
|
||||
return struct.pack(">I", 8 + len(payload)) + kind + payload
|
||||
|
||||
|
||||
def _full_box(kind, *parts):
|
||||
return _box(kind, b"\0\0\0\0", *parts)
|
||||
|
||||
|
||||
def _mdta_udta(tags):
|
||||
"""``moov/udta/meta`` with one mdta key per tag, laid out as ffmpeg writes it."""
|
||||
names = list(tags)
|
||||
keys = _full_box(
|
||||
b"keys",
|
||||
struct.pack(">I", len(names)),
|
||||
*(
|
||||
struct.pack(">I", 8 + len(name.encode("utf-8")))
|
||||
+ b"mdta"
|
||||
+ name.encode("utf-8")
|
||||
for name in names
|
||||
),
|
||||
)
|
||||
items = b"".join(
|
||||
_box(
|
||||
struct.pack(">I", index + 1),
|
||||
_box(b"data", struct.pack(">II", 1, 0), tags[name].encode("utf-8")),
|
||||
)
|
||||
for index, name in enumerate(names)
|
||||
)
|
||||
hdlr = _full_box(b"hdlr", b"\0\0\0\0", b"mdta", bytes(12), b"\0")
|
||||
return _box(b"udta", _full_box(b"meta", hdlr, keys, _box(b"ilst", items)))
|
||||
|
||||
|
||||
def _with_udta(data, udta):
|
||||
"""``data`` (an MP4 with moov after mdat) with its moov's udta replaced."""
|
||||
out = bytearray()
|
||||
offset = 0
|
||||
while offset < len(data):
|
||||
size, kind = struct.unpack(">I4s", data[offset : offset + 8])
|
||||
if kind != b"moov":
|
||||
out += data[offset : offset + size]
|
||||
offset += size
|
||||
continue
|
||||
body = data[offset + 8 : offset + size]
|
||||
children = bytearray()
|
||||
inner = 0
|
||||
while inner < len(body):
|
||||
child_size, child_kind = struct.unpack(">I4s", body[inner : inner + 8])
|
||||
if child_kind != b"udta":
|
||||
children += body[inner : inner + child_size]
|
||||
inner += child_size
|
||||
children += udta
|
||||
out += struct.pack(">I", 8 + len(children)) + b"moov" + bytes(children)
|
||||
offset += size
|
||||
return bytes(out)
|
||||
|
||||
|
||||
def _text_of(size, prefix):
|
||||
"""A valid JSON object of exactly ``size`` UTF-8 bytes with quotes throughout."""
|
||||
unit = '"k":"v",'
|
||||
body = (unit * (size // len(unit) + 1))[: size - len(prefix) - 2]
|
||||
body = body[: body.rfind(",")] if body.endswith(",") else body
|
||||
text = prefix + body + "}"
|
||||
return text + " " * (size - len(text))
|
||||
|
||||
|
||||
class LocalPngReaderParityTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
@@ -199,24 +498,9 @@ class LocalPngReaderParityTests(unittest.TestCase):
|
||||
return files
|
||||
|
||||
def _oracle(self, files):
|
||||
node = shutil.which("node")
|
||||
if node is None: # pragma: no cover - CI runners and dev hosts ship Node
|
||||
if os.environ.get("CI"):
|
||||
self.fail("Node.js is required in CI for the local reader parity test")
|
||||
self.skipTest("Node.js is not installed")
|
||||
payload = json.dumps(
|
||||
[{"name": name, "path": str(path)} for name, path in files.items()]
|
||||
return _run_oracle(
|
||||
self, [{"name": name, "path": str(path)} for name, path in files.items()]
|
||||
)
|
||||
# Fixed local Node harness, no shell.
|
||||
completed = subprocess.run( # nosec B603
|
||||
[node, str(ORACLE)],
|
||||
input=payload.encode("utf-8"),
|
||||
capture_output=True,
|
||||
timeout=NODE_TIMEOUT_SECONDS,
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(completed.returncode, 0, completed.stderr.decode("utf-8"))
|
||||
return json.loads(completed.stdout)
|
||||
|
||||
def test_local_png_reader_matches_read_inspection_file(self):
|
||||
files = self._files()
|
||||
@@ -272,5 +556,139 @@ class LocalPngReaderParityTests(unittest.TestCase):
|
||||
self.assertLess(path.stat().st_size, 4096)
|
||||
|
||||
|
||||
class LocalVideoCarrierParityTests(unittest.TestCase):
|
||||
"""The local video readers and carrier rule answer as the server does."""
|
||||
|
||||
def setUp(self):
|
||||
temporary = tempfile.TemporaryDirectory()
|
||||
self.addCleanup(temporary.cleanup)
|
||||
self.addCleanup(assert_closed, self, temporary.name)
|
||||
self.directory = Path(temporary.name)
|
||||
|
||||
def test_local_carrier_matches_server_selection(self):
|
||||
vectors = [(name, tags, []) for name, tags in RECORDED_TAGS.items()]
|
||||
vectors += TAG_MAP_VECTORS
|
||||
observed = _run_oracle(
|
||||
self,
|
||||
[
|
||||
{"name": name, "tags": tags, "limited": limited}
|
||||
for name, tags, limited in vectors
|
||||
],
|
||||
)
|
||||
outcomes = set()
|
||||
for name, tags, limited in vectors:
|
||||
with self.subTest(vector=name):
|
||||
selection = observed[name]["result"]["selection"]
|
||||
server = _server_selection(tags, limited)
|
||||
self.assertEqual(
|
||||
_local_shape(selection), server, selection.get("reason")
|
||||
)
|
||||
if server is None:
|
||||
outcomes.add(selection["status"] + ":" + selection["reason"])
|
||||
elif not server["records"] and not server["record_formats"]:
|
||||
outcomes.add("no-carrier")
|
||||
else:
|
||||
outcomes.update(server["record_formats"].values())
|
||||
# Every outcome of the rule must occur, or agreement proves little.
|
||||
self.assertEqual(
|
||||
outcomes,
|
||||
{
|
||||
"original-text",
|
||||
"reserialized-json",
|
||||
"invalid",
|
||||
"limited",
|
||||
"no-carrier",
|
||||
"unreadable:duplicate-metadata",
|
||||
"unreadable:invalid-metadata",
|
||||
"limited:metadata-limit",
|
||||
},
|
||||
)
|
||||
|
||||
def test_local_video_readers_match_recorded_tags_and_server(self):
|
||||
files = {name: VIDEO_FIXTURES / name for name in RECORDED_TAGS}
|
||||
observed = _run_oracle(
|
||||
self, [{"name": name, "path": str(path)} for name, path in files.items()]
|
||||
)
|
||||
for name, path in files.items():
|
||||
with self.subTest(fixture=name):
|
||||
self.assertLess(path.stat().st_size, 16 * 1024)
|
||||
container = VIDEO_CONTAINERS[path.suffix]
|
||||
self.assertEqual(
|
||||
observed[name]["classified"], {"media": "video", "kind": container}
|
||||
)
|
||||
read = observed[name]["result"]["read"]
|
||||
self.assertEqual(read["status"], "complete", read)
|
||||
self.assertEqual(read["container"], container)
|
||||
self.assertEqual(read["tags"], _known_tags(RECORDED_TAGS[name]))
|
||||
self.assertEqual(read["limited"], [])
|
||||
local = _local_shape(observed[name]["result"]["selection"])
|
||||
self.assertEqual(local, _server_selection(RECORDED_TAGS[name]))
|
||||
if HAVE_FFMPEG:
|
||||
self.assertEqual(local, _read_shape(read_inspection_file(path)))
|
||||
# The corpus covers per-key carriers in every container, an envelope and a
|
||||
# Matroska COMMENT that the exact-case envelope rule leaves unread.
|
||||
self.assertTrue(
|
||||
_server_selection(RECORDED_TAGS["legacy-comment.mp4"])["carrierText"]
|
||||
)
|
||||
self.assertEqual(
|
||||
_server_selection(RECORDED_TAGS["legacy-comment.webm"])["records"], {}
|
||||
)
|
||||
|
||||
@unittest.skipUnless(HAVE_FFMPEG, "ffmpeg/ffprobe not installed")
|
||||
def test_large_per_key_videos_read_the_same_as_the_server(self):
|
||||
base = self.directory / "base.mp4"
|
||||
write_video(base)
|
||||
source = base.read_bytes()
|
||||
shapes = {
|
||||
"available": {
|
||||
"prompt": _text_of(
|
||||
MiB, '{"1":{"class_type":"Note","inputs":{"text":"a"}},'
|
||||
),
|
||||
"workflow": _text_of(5 * MiB // 2, '{"nodes":[],"version":0.4,'),
|
||||
},
|
||||
"workflow_limited": {
|
||||
"prompt": GRAPH,
|
||||
"workflow": "[" + "0," * (WORKFLOW_METADATA_BYTES // 2) + "]",
|
||||
},
|
||||
"roles_over_budget": {"prompt": "A" * (ROLE_METADATA_BYTES + 1)},
|
||||
}
|
||||
files = {}
|
||||
for name, tags in shapes.items():
|
||||
path = self.directory / f"{name}.mp4"
|
||||
path.write_bytes(_with_udta(source, _mdta_udta(tags)))
|
||||
files[name] = path
|
||||
observed = _run_oracle(
|
||||
self, [{"name": name, "path": str(path)} for name, path in files.items()]
|
||||
)
|
||||
available = observed["available"]["result"]
|
||||
self.assertEqual(available["read"]["status"], "complete")
|
||||
self.assertEqual(available["read"]["limited"], [])
|
||||
self.assertEqual(
|
||||
available["selection"]["records"], shapes["available"], "1 MiB + 2.5 MiB"
|
||||
)
|
||||
self.assertEqual(
|
||||
_local_shape(available["selection"]),
|
||||
_read_shape(read_inspection_file(files["available"])),
|
||||
)
|
||||
limited = observed["workflow_limited"]["result"]
|
||||
self.assertEqual(limited["read"]["limited"], ["workflow"])
|
||||
self.assertNotIn("workflow", limited["read"]["tags"])
|
||||
self.assertEqual(
|
||||
_local_shape(limited["selection"]),
|
||||
_read_shape(read_inspection_file(files["workflow_limited"])),
|
||||
)
|
||||
self.assertEqual(
|
||||
limited["selection"]["record_formats"],
|
||||
{"prompt": "original-text", "workflow": "limited"},
|
||||
)
|
||||
over = observed["roles_over_budget"]["result"]
|
||||
self.assertEqual(
|
||||
over["read"], {"status": "limited", "reason": "metadata-limit"}
|
||||
)
|
||||
self.assertIsNone(over["selection"])
|
||||
with self.assertRaises(MetadataReadError):
|
||||
read_inspection_file(files["roles_over_budget"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -390,5 +390,44 @@ class TestReadVideoMetadataWithRealFfprobe(unittest.TestCase):
|
||||
self.assertEqual(read_video_metadata(path), {})
|
||||
|
||||
|
||||
@unittest.skipUnless(HAVE_FFMPEG, "ffmpeg/ffprobe not installed")
|
||||
class VideoFixtureFfprobeTests(unittest.TestCase):
|
||||
"""Characterization: the committed local-inspect video fixtures still carry the
|
||||
tag maps recorded in ``video-tags-v1.json``, which the browser reader parity
|
||||
tests use in place of ffprobe. A different ffprobe that reports other names or
|
||||
values for the same bytes shows up here, not as a silent parity gap."""
|
||||
|
||||
FIXTURES = os.path.join(
|
||||
os.path.dirname(os.path.abspath(__file__)), "fixtures", "local-inspect"
|
||||
)
|
||||
|
||||
def test_committed_fixtures_produce_their_recorded_tags(self):
|
||||
with open(
|
||||
os.path.join(self.FIXTURES, "video-tags-v1.json"), encoding="utf-8"
|
||||
) as stream:
|
||||
recorded = json.load(stream)["fixtures"]
|
||||
self.assertEqual(len(recorded), 8)
|
||||
for name, tags in recorded.items():
|
||||
with self.subTest(fixture=name):
|
||||
probe = subprocess.run(
|
||||
[
|
||||
shutil.which("ffprobe"),
|
||||
"-v",
|
||||
"quiet",
|
||||
"-show_entries",
|
||||
"format_tags",
|
||||
"-of",
|
||||
"json",
|
||||
os.path.join(self.FIXTURES, name),
|
||||
],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
encoding="utf-8",
|
||||
check=True,
|
||||
timeout=60,
|
||||
)
|
||||
self.assertEqual(json.loads(probe.stdout)["format"]["tags"], tags)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
+7
-1
@@ -73,7 +73,7 @@
|
||||
},
|
||||
"png-metadata.js": {
|
||||
"lines": 98,
|
||||
"branches": 90,
|
||||
"branches": 96,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
@@ -293,6 +293,12 @@
|
||||
"functions": 96,
|
||||
"statements": 96
|
||||
},
|
||||
"workspace/local-video.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 93,
|
||||
"functions": 98,
|
||||
"statements": 98
|
||||
},
|
||||
"workspace/media-viewer.mjs": {
|
||||
"lines": 98,
|
||||
"branches": 93,
|
||||
|
||||
@@ -0,0 +1,896 @@
|
||||
// Local video tag readers for the Inspect page (#249): bounded ISO-BMFF (MP4/MOV/M4V)
|
||||
// and Matroska (MKV/WebM) walkers that read only the container tags the server asks
|
||||
// ffprobe for (utils/prompt_inspector.py _video_metadata), and a port of the carrier
|
||||
// rule utils/video_metadata.py select_video_carrier plus the record budgets of
|
||||
// read_inspection_file. Nothing here uploads, stores or logs the file. The walkers
|
||||
// slice box and element headers, the tag names and the retained tag values only, so
|
||||
// a multi-gigabyte video costs at most VIDEO_READ_CEILING bytes of reads and never a
|
||||
// byte inside mdat or a Cluster. tests/test_local_inspect_parity.py proves the
|
||||
// answers equal the server's on the committed fixtures and recorded tag maps.
|
||||
//
|
||||
// What ffprobe reports is the oracle: ISO-BMFF mdta keys are reported verbatim and
|
||||
// the legacy ilst atoms ©cmt/desc as comment/description; Matroska TagNames are
|
||||
// reported as written (ffmpeg upper-cases them at mux time). Envelope names are
|
||||
// exact-case (comment, description) and the per-key names fold ASCII case in the
|
||||
// selector, exactly as the server folds them.
|
||||
|
||||
// Role records (prompt, parameters, the pair execution record) share one budget:
|
||||
// past it the read fails. A workflow has its own budget; past it only the workflow
|
||||
// is "limited". Duplicate tags draw from separate budgets per class. The total of
|
||||
// tag bytes read is the server's ffprobe stdout bound.
|
||||
export const ROLE_METADATA_BYTES = 2 * 1024 * 1024;
|
||||
export const WORKFLOW_METADATA_BYTES = 4 * 1024 * 1024;
|
||||
export const DUPLICATE_WORKFLOW_BYTES = 2 * WORKFLOW_METADATA_BYTES;
|
||||
export const DUPLICATE_ROLE_BYTES = 2 * ROLE_METADATA_BYTES;
|
||||
export const VIDEO_TAG_READ_BYTES = 2 * (ROLE_METADATA_BYTES + WORKFLOW_METADATA_BYTES) + 64 * 1024;
|
||||
export const MAX_VIDEO_ELEMENTS = 10000;
|
||||
const MAX_HEADER_BYTES = 16; // a 64-bit ISO-BMFF box header; an EBML header is at most 12
|
||||
const MAX_NAME_BYTES = 64; // the longest carrier name is 24 bytes; longer names are never ours
|
||||
// Every byte the readers can slice from a video of any size: each element's header
|
||||
// and name plus the retained tag values.
|
||||
export const VIDEO_READ_CEILING = MAX_VIDEO_ELEMENTS * (MAX_HEADER_BYTES + MAX_NAME_BYTES) + VIDEO_TAG_READ_BYTES;
|
||||
const PREFIX_BYTES = 32;
|
||||
const YIELD_INTERVAL_MS = 8;
|
||||
const JSON_MAX_DEPTH = 1000; // Python's default recursion limit
|
||||
|
||||
const ROLE_KEYS = ["prompt", "parameters", "comfyassets_prompt_pairs"];
|
||||
const METADATA_KEYS = ["prompt", "workflow", "parameters", "comfyassets_prompt_pairs"];
|
||||
const ENVELOPE_KEYS = ["comment", "description"];
|
||||
const PER_KEY_NAMES = new Set(METADATA_KEYS);
|
||||
const ENVELOPE_NAMES = new Set(ENVELOPE_KEYS);
|
||||
// ffprobe's names for the legacy ilst atoms that can carry an envelope.
|
||||
const LEGACY_ATOMS = new Map([["©cmt", "comment"], ["desc", "description"], ["©des", "description"]]);
|
||||
const ASF_GUID = [0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c];
|
||||
|
||||
const EBML = {
|
||||
header: 0x1a45dfa3,
|
||||
docType: 0x4282,
|
||||
segment: 0x18538067,
|
||||
seekHead: 0x114d9b74,
|
||||
seek: 0x4dbb,
|
||||
seekId: 0x53ab,
|
||||
seekPosition: 0x53ac,
|
||||
tags: 0x1254c367,
|
||||
tag: 0x7373,
|
||||
targets: 0x63c0,
|
||||
trackUid: 0x63c5,
|
||||
chapterUid: 0x63c4,
|
||||
attachmentUid: 0x63c6,
|
||||
simpleTag: 0x67c8,
|
||||
tagName: 0x45a3,
|
||||
tagString: 0x4487,
|
||||
tagLanguage: 0x447a,
|
||||
tagDefault: 0x4484,
|
||||
cluster: 0x1f43b675,
|
||||
};
|
||||
const TAGS_ID_BYTES = [0x12, 0x54, 0xc3, 0x67];
|
||||
|
||||
class VideoReadError extends Error {
|
||||
constructor(status, reason, message) {
|
||||
super(message);
|
||||
this.name = "VideoReadError";
|
||||
this.status = status;
|
||||
this.reason = reason;
|
||||
}
|
||||
}
|
||||
|
||||
const invalid = (message) => new VideoReadError("unreadable", "invalid-container", message);
|
||||
const limited = (reason, message) => new VideoReadError("limited", reason, message);
|
||||
const readFailure = (message) => new VideoReadError("unreadable", "read-failure", message);
|
||||
|
||||
function abortError() {
|
||||
return new DOMException("The file read was aborted", "AbortError");
|
||||
}
|
||||
|
||||
function throwIfAborted(signal) {
|
||||
if (signal && signal.aborted) throw abortError();
|
||||
}
|
||||
|
||||
function toBytes(input) {
|
||||
if (input instanceof Uint8Array) return input;
|
||||
if (input instanceof ArrayBuffer) return new Uint8Array(input);
|
||||
if (Array.isArray(input)) return Uint8Array.from(input);
|
||||
return null;
|
||||
}
|
||||
|
||||
function latin1(bytes, start, end) {
|
||||
let out = "";
|
||||
for (let index = start; index < end; index += 1) out += String.fromCharCode(bytes[index]);
|
||||
return out;
|
||||
}
|
||||
|
||||
function startsWith(bytes, prefix, at = 0) {
|
||||
if (bytes.length < at + prefix.length) return false;
|
||||
for (let index = 0; index < prefix.length; index += 1) if (bytes[at + index] !== prefix[index]) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
function readUint32(bytes, offset) {
|
||||
return ((bytes[offset] << 24) >>> 0) + (bytes[offset + 1] << 16) + (bytes[offset + 2] << 8) + bytes[offset + 3];
|
||||
}
|
||||
|
||||
/** A big-endian unsigned integer of up to 8 bytes as a Number, or null past MAX_SAFE_INTEGER. */
|
||||
function readUintBytes(bytes, start, end) {
|
||||
let value = 0n;
|
||||
for (let index = start; index < end; index += 1) value = (value << 8n) | BigInt(bytes[index]);
|
||||
return value <= BigInt(Number.MAX_SAFE_INTEGER) ? Number(value) : null;
|
||||
}
|
||||
|
||||
function concatBytes(parts, total) {
|
||||
const out = new Uint8Array(total);
|
||||
let offset = 0;
|
||||
for (const part of parts) {
|
||||
out.set(part, offset);
|
||||
offset += part.length;
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** ffmpeg stores tag text as a C string: the first NUL ends it. */
|
||||
function untilNul(bytes) {
|
||||
const nul = bytes.indexOf(0);
|
||||
return nul < 0 ? bytes : bytes.subarray(0, nul);
|
||||
}
|
||||
|
||||
/** Strict UTF-8, as Python decodes ffprobe's output: invalid bytes fail the read. */
|
||||
function utf8Strict(bytes) {
|
||||
try {
|
||||
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
|
||||
} catch (error) {
|
||||
throw invalid("Video metadata is not valid UTF-8");
|
||||
}
|
||||
}
|
||||
|
||||
/** UTF-8 length as Python's ``encode("utf-8", "surrogatepass")`` counts it. */
|
||||
function utf8Length(text) {
|
||||
let total = 0;
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const code = text.charCodeAt(index);
|
||||
if (code < 0x80) total += 1;
|
||||
else if (code < 0x800) total += 2;
|
||||
else if (code >= 0xd800 && code <= 0xdbff && index + 1 < text.length) {
|
||||
const next = text.charCodeAt(index + 1);
|
||||
if (next >= 0xdc00 && next <= 0xdfff) {
|
||||
total += 4;
|
||||
index += 1;
|
||||
} else total += 3;
|
||||
} else total += 3;
|
||||
}
|
||||
return total;
|
||||
}
|
||||
|
||||
function defaultYield() {
|
||||
return new Promise((resolve) => setTimeout(resolve, 0));
|
||||
}
|
||||
|
||||
/** Hands the event loop a turn about every YIELD_INTERVAL_MS of work. */
|
||||
function createPacer(yieldControl, signal) {
|
||||
let last = performance.now();
|
||||
return {
|
||||
async maybeYield() {
|
||||
if (performance.now() - last < YIELD_INTERVAL_MS) return;
|
||||
await yieldControl();
|
||||
throwIfAborted(signal);
|
||||
last = performance.now();
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Logical byte ranges of `file`, served from the retained `prefix` where it covers
|
||||
* them (the twin of png-metadata.js's reader). Every sliced byte is charged against
|
||||
* VIDEO_READ_CEILING, and every range is checked against `file.size` before slicing.
|
||||
*/
|
||||
function createSliceReader(file, prefix, signal) {
|
||||
const retained = prefix || new Uint8Array(0);
|
||||
let charged = 0;
|
||||
return {
|
||||
async read(start, end) {
|
||||
throwIfAborted(signal);
|
||||
if (!Number.isSafeInteger(end) || end > file.size || start > end) throw invalid("Incomplete video container");
|
||||
if (end <= retained.length) return retained.subarray(start, end);
|
||||
const head = start < retained.length ? retained.subarray(start) : null;
|
||||
const sliceStart = head ? retained.length : start;
|
||||
charged += end - sliceStart;
|
||||
if (charged > VIDEO_READ_CEILING) throw readFailure("Video read exceeds the inspection ceiling");
|
||||
const buffer = await file.slice(sliceStart, end).arrayBuffer();
|
||||
throwIfAborted(signal);
|
||||
const bytes = new Uint8Array(buffer);
|
||||
if (bytes.length !== end - sliceStart) throw readFailure("The file changed while it was being read");
|
||||
return head ? concatBytes([head, bytes], end - start) : bytes;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/** Container kind from the first bytes, mirroring png-metadata.js classifyPrefix. */
|
||||
function detectKind(bytes) {
|
||||
if (!bytes) return null;
|
||||
if (bytes.length >= 12 && latin1(bytes, 0, 4) === "RIFF") return latin1(bytes, 8, 12) === "AVI " ? "avi" : null;
|
||||
if (bytes.length >= 8 && latin1(bytes, 4, 8) === "ftyp") return "iso-bmff";
|
||||
if (startsWith(bytes, [0x1a, 0x45, 0xdf, 0xa3])) return "matroska";
|
||||
if (startsWith(bytes, ASF_GUID)) return "asf";
|
||||
return null;
|
||||
}
|
||||
|
||||
/**
|
||||
* The shared walk state: the element count, the tag budgets and the collected tags.
|
||||
* `value` is the one gate every tag value passes before it is sliced.
|
||||
*/
|
||||
function createWalk(reader, signal, pacer) {
|
||||
const tags = {};
|
||||
const limitedRecords = [];
|
||||
const seen = new Set();
|
||||
let count = 0;
|
||||
let role = 0;
|
||||
let workflow = 0;
|
||||
let duplicateRole = 0;
|
||||
let duplicateWorkflow = 0;
|
||||
let total = 0;
|
||||
const overLimit = () => limited("metadata-limit", "Video metadata exceeds inspection limits");
|
||||
return {
|
||||
tags,
|
||||
limited: limitedRecords,
|
||||
async header(offset, length) {
|
||||
count += 1;
|
||||
if (count > MAX_VIDEO_ELEMENTS) throw limited("chunk-limit", "Video container exceeds inspection limits");
|
||||
await pacer.maybeYield();
|
||||
return reader.read(offset, offset + length);
|
||||
},
|
||||
read: (start, end) => reader.read(start, end),
|
||||
/** Whether `name` is a carrier the server reads (per-key folded, envelope exact). */
|
||||
known(name) {
|
||||
return PER_KEY_NAMES.has(name.toLowerCase()) || ENVELOPE_NAMES.has(name);
|
||||
},
|
||||
remove(name) {
|
||||
delete tags[name];
|
||||
},
|
||||
/**
|
||||
* Reads and keeps the tag `name` (a known carrier, see `known`) whose value
|
||||
* spans [start, end), within the budgets. `join` appends to an existing value
|
||||
* with that separator (ISO-BMFF repeats, as ffprobe reports them); `text`
|
||||
* stores an already decoded value.
|
||||
*/
|
||||
async value(name, start, end, { join = null, text = null } = {}) {
|
||||
const size = end - start;
|
||||
const folded = name.toLowerCase();
|
||||
if (PER_KEY_NAMES.has(folded)) {
|
||||
const first = !seen.has(folded);
|
||||
seen.add(folded);
|
||||
if (folded === "workflow") {
|
||||
if (first && size > WORKFLOW_METADATA_BYTES) {
|
||||
// The server's role-only re-probe: the per-key workflow is left out.
|
||||
if (!limitedRecords.includes("workflow")) limitedRecords.push("workflow");
|
||||
return;
|
||||
}
|
||||
if (first) workflow += size;
|
||||
else duplicateWorkflow += size;
|
||||
if (duplicateWorkflow > DUPLICATE_WORKFLOW_BYTES) throw overLimit();
|
||||
} else {
|
||||
if (first) role += size;
|
||||
else duplicateRole += size;
|
||||
if (role > ROLE_METADATA_BYTES || duplicateRole > DUPLICATE_ROLE_BYTES) throw overLimit();
|
||||
}
|
||||
}
|
||||
total += size;
|
||||
if (total > VIDEO_TAG_READ_BYTES) throw overLimit();
|
||||
const value = text ?? utf8Strict(untilNul(await reader.read(start, end)));
|
||||
tags[name] = join !== null && name in tags ? `${tags[name]}${join}${value}` : value;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ ISO-BMFF
|
||||
|
||||
/** One box header at `offset` inside a parent ending at `parentEnd`, or null when fewer than 8 bytes remain. */
|
||||
async function readBox(walk, offset, parentEnd) {
|
||||
if (parentEnd - offset < 8) return null;
|
||||
const bytes = await walk.header(offset, Math.min(MAX_HEADER_BYTES, parentEnd - offset));
|
||||
let size = readUint32(bytes, 0);
|
||||
let bodyStart = offset + 8;
|
||||
if (size === 1) {
|
||||
if (bytes.length < 16) throw invalid("Truncated 64-bit box size");
|
||||
const large = readUintBytes(bytes, 8, 16);
|
||||
if (large === null || large < 16) throw invalid("Invalid 64-bit box size");
|
||||
size = large;
|
||||
bodyStart = offset + 16;
|
||||
} else if (size === 0) {
|
||||
size = parentEnd - offset;
|
||||
} else if (size < 8) {
|
||||
throw invalid("Box size below its header");
|
||||
}
|
||||
const end = offset + size;
|
||||
if (end > parentEnd) throw invalid("Box extends past its parent");
|
||||
return { type: latin1(bytes, 4, 8), raw: bytes.subarray(4, 8), start: offset, bodyStart, end };
|
||||
}
|
||||
|
||||
async function readIsoBmff(walk, fileSize) {
|
||||
const state = { hdlrMdta: false, keys: null, format: null };
|
||||
let offset = 0;
|
||||
let box;
|
||||
while ((box = await readBox(walk, offset, fileSize))) {
|
||||
if (box.type === "ftyp" && box.end - box.bodyStart >= 4) {
|
||||
state.format = latin1(await walk.read(box.bodyStart, box.bodyStart + 4), 0, 4).replace(/[\s\0]+$/, "");
|
||||
} else if (box.type === "moov") {
|
||||
await readMoov(walk, box, state);
|
||||
return state.format;
|
||||
}
|
||||
offset = box.end;
|
||||
}
|
||||
throw invalid("No moov box");
|
||||
}
|
||||
|
||||
async function readMoov(walk, moov, state) {
|
||||
let offset = moov.bodyStart;
|
||||
let box;
|
||||
while ((box = await readBox(walk, offset, moov.end))) {
|
||||
if (box.type === "meta") await readMeta(walk, box, state);
|
||||
else if (box.type === "udta") {
|
||||
let inner = box.bodyStart;
|
||||
let child;
|
||||
while ((child = await readBox(walk, inner, box.end))) {
|
||||
if (child.type === "meta") await readMeta(walk, child, state);
|
||||
inner = child.end;
|
||||
}
|
||||
}
|
||||
offset = box.end;
|
||||
}
|
||||
}
|
||||
|
||||
/** `meta` as ffmpeg reads it: scan 4-byte words for the first child's `hdlr` type. */
|
||||
async function readMeta(walk, meta, state) {
|
||||
const scan = await walk.read(meta.bodyStart, Math.min(meta.end, meta.bodyStart + MAX_NAME_BYTES));
|
||||
let childrenStart = null;
|
||||
for (let word = 4; word + 4 <= scan.length; word += 4) {
|
||||
if (latin1(scan, word, word + 4) === "hdlr") {
|
||||
childrenStart = meta.bodyStart + word - 4;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (childrenStart === null) return;
|
||||
let offset = childrenStart;
|
||||
let box;
|
||||
while ((box = await readBox(walk, offset, meta.end))) {
|
||||
if (box.type === "hdlr") {
|
||||
if (box.end - box.bodyStart >= 12) {
|
||||
const handler = await walk.read(box.bodyStart + 8, box.bodyStart + 12);
|
||||
if (latin1(handler, 0, 4) === "mdta") state.hdlrMdta = true;
|
||||
}
|
||||
} else if (box.type === "keys") await readKeys(walk, box, state);
|
||||
else if (box.type === "ilst") await readIlst(walk, box, state);
|
||||
offset = box.end;
|
||||
}
|
||||
}
|
||||
|
||||
/** The `keys` box: 1-based mdta names; other namespaces and overlong names address nothing. */
|
||||
async function readKeys(walk, box, state) {
|
||||
if (box.end - box.bodyStart < 8) return;
|
||||
const head = await walk.read(box.bodyStart, box.bodyStart + 8);
|
||||
const count = readUint32(head, 4);
|
||||
const keys = [null];
|
||||
let offset = box.bodyStart + 8;
|
||||
for (let index = 1; index <= count; index += 1) {
|
||||
if (box.end - offset < 8) throw invalid("Keys box count exceeds its entries");
|
||||
const entry = await walk.header(offset, 8);
|
||||
const keySize = readUint32(entry, 0);
|
||||
if (keySize < 8 || offset + keySize > box.end) throw invalid("Keys entry exceeds its box");
|
||||
const nameLength = keySize - 8;
|
||||
let name = null;
|
||||
if (latin1(entry, 4, 8) === "mdta" && nameLength <= MAX_NAME_BYTES) {
|
||||
name = utf8Strict(await walk.read(offset + 8, offset + keySize));
|
||||
}
|
||||
keys.push(name);
|
||||
offset += keySize;
|
||||
}
|
||||
state.keys = keys;
|
||||
}
|
||||
|
||||
/** ffprobe's text for an integer `data` atom (types 21 and 22) of 1, 2, 4 or 8 bytes, else null. */
|
||||
function integerText(bytes, signed) {
|
||||
if (![1, 2, 4, 8].includes(bytes.length)) return null;
|
||||
let value = 0n;
|
||||
for (const byte of bytes) value = (value << 8n) | BigInt(byte);
|
||||
if (signed) value = BigInt.asIntN(bytes.length * 8, value);
|
||||
return value.toString();
|
||||
}
|
||||
|
||||
/**
|
||||
* ilst items: a legacy fourcc or a `keys` index, then the leading run of `data`
|
||||
* children. As ffprobe reports them, repeated values of one name join with ";",
|
||||
* types 0, 1 and 4 are text, 21 and 22 are integers, other types are not tags.
|
||||
*/
|
||||
async function readIlst(walk, ilst, state) {
|
||||
let offset = ilst.bodyStart;
|
||||
let item;
|
||||
while ((item = await readBox(walk, offset, ilst.end))) {
|
||||
offset = item.end;
|
||||
let name = LEGACY_ATOMS.get(item.type);
|
||||
if (name === undefined && state.hdlrMdta && state.keys) {
|
||||
const index = readUint32(item.raw, 0);
|
||||
if (index === 0 || index >= state.keys.length) throw invalid("ilst index out of range");
|
||||
name = state.keys[index];
|
||||
}
|
||||
if (!name || !walk.known(name)) continue;
|
||||
let inner = item.bodyStart;
|
||||
while (item.end - inner >= 16) {
|
||||
const data = await walk.header(inner, 16);
|
||||
const dataSize = readUint32(data, 0);
|
||||
if (latin1(data, 4, 8) !== "data" || dataSize < 16 || dataSize > item.end - inner) break;
|
||||
const type = readUint32(data, 8);
|
||||
const start = inner + 16;
|
||||
const end = inner + dataSize;
|
||||
inner = end;
|
||||
if (type === 0 || type === 1 || type === 4) await walk.value(name, start, end, { join: ";" });
|
||||
else if ((type === 21 || type === 22) && end - start <= 8) {
|
||||
const text = integerText(await walk.read(start, end), type === 21);
|
||||
if (text !== null) await walk.value(name, start, end, { join: ";", text });
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------ Matroska
|
||||
|
||||
/** One EBML element header at `offset` inside a parent ending at `parentEnd`. */
|
||||
async function readElement(walk, offset, parentEnd) {
|
||||
if (parentEnd - offset < 2) return null;
|
||||
const bytes = await walk.header(offset, Math.min(12, parentEnd - offset));
|
||||
const idLength = Math.clz32(bytes[0]) - 23;
|
||||
if (bytes[0] === 0 || idLength > 4 || bytes.length < idLength + 1) throw invalid("Invalid EBML element id");
|
||||
const id = readUintBytes(bytes, 0, idLength);
|
||||
const sizeByte = bytes[idLength];
|
||||
const sizeLength = Math.clz32(sizeByte) - 23;
|
||||
if (sizeByte === 0 || bytes.length < idLength + sizeLength) throw invalid("Invalid EBML element size");
|
||||
let size = BigInt(sizeByte & (0xff >> sizeLength));
|
||||
for (let index = idLength + 1; index < idLength + sizeLength; index += 1) size = (size << 8n) | BigInt(bytes[index]);
|
||||
const unknown = size === (1n << BigInt(7 * sizeLength)) - 1n;
|
||||
if (!unknown && size > BigInt(Number.MAX_SAFE_INTEGER)) throw invalid("EBML element size exceeds inspection limits");
|
||||
const dataStart = offset + idLength + sizeLength;
|
||||
const end = unknown ? null : dataStart + Number(size);
|
||||
if (end !== null && end > parentEnd) throw invalid("EBML element extends past its parent");
|
||||
return { id, dataStart, end, unknown };
|
||||
}
|
||||
|
||||
/** Children of a sized master element, in order. */
|
||||
async function* children(walk, element) {
|
||||
let offset = element.dataStart;
|
||||
let child;
|
||||
while ((child = await readElement(walk, offset, element.end))) {
|
||||
if (child.unknown) throw invalid("Unknown-size element inside a master element");
|
||||
yield child;
|
||||
offset = child.end;
|
||||
}
|
||||
}
|
||||
|
||||
async function readMatroska(walk, fileSize) {
|
||||
const header = await readElement(walk, 0, fileSize);
|
||||
if (!header || header.id !== EBML.header || header.unknown) throw invalid("Not an EBML file");
|
||||
let docType = null;
|
||||
for await (const child of children(walk, header)) {
|
||||
if (child.id === EBML.docType && child.end - child.dataStart <= MAX_NAME_BYTES) {
|
||||
docType = latin1(untilNul(await walk.read(child.dataStart, child.end)), 0, Math.min(child.end - child.dataStart, MAX_NAME_BYTES));
|
||||
}
|
||||
}
|
||||
if (docType !== "matroska" && docType !== "webm") throw invalid("Unsupported EBML DocType");
|
||||
const segment = await readElement(walk, header.end, fileSize);
|
||||
if (!segment || segment.id !== EBML.segment) throw invalid("No Segment element");
|
||||
const segmentEnd = segment.unknown ? fileSize : segment.end;
|
||||
// As ffmpeg's demuxer: level-1 elements are read up to the first Cluster (of known
|
||||
// or unknown size), then SeekHead entries locate Tags stored behind the clusters.
|
||||
const seekTargets = [];
|
||||
let offset = segment.dataStart;
|
||||
let element;
|
||||
while ((element = await readElement(walk, offset, segmentEnd))) {
|
||||
if (element.id === EBML.cluster) break;
|
||||
if (element.unknown) throw invalid("Unknown-size element that is not a Segment or Cluster");
|
||||
if (element.id === EBML.seekHead) await readSeekHead(walk, element, segment.dataStart, seekTargets);
|
||||
else if (element.id === EBML.tags) await readTags(walk, element);
|
||||
offset = element.end;
|
||||
}
|
||||
for (const target of seekTargets) {
|
||||
if (target < offset || target >= segmentEnd) continue;
|
||||
const pointed = await readElement(walk, target, segmentEnd);
|
||||
if (!pointed || pointed.id !== EBML.tags || pointed.unknown) continue;
|
||||
await readTags(walk, pointed);
|
||||
}
|
||||
return docType;
|
||||
}
|
||||
|
||||
async function readSeekHead(walk, seekHead, segmentStart, targets) {
|
||||
for await (const seek of children(walk, seekHead)) {
|
||||
if (seek.id !== EBML.seek) continue;
|
||||
let isTags = false;
|
||||
let position = null;
|
||||
for await (const field of children(walk, seek)) {
|
||||
const length = field.end - field.dataStart;
|
||||
if (field.id === EBML.seekId && length <= 4) {
|
||||
isTags = startsWith(await walk.read(field.dataStart, field.end), TAGS_ID_BYTES) && length === 4;
|
||||
} else if (field.id === EBML.seekPosition && length <= 8) {
|
||||
position = readUintBytes(await walk.read(field.dataStart, field.end), 0, length);
|
||||
}
|
||||
}
|
||||
if (isTags && position !== null) targets.push(segmentStart + position);
|
||||
}
|
||||
}
|
||||
|
||||
async function readTags(walk, tags) {
|
||||
for await (const tag of children(walk, tags)) {
|
||||
if (tag.id !== EBML.tag) continue;
|
||||
let targeted = false;
|
||||
const simpleTags = [];
|
||||
for await (const child of children(walk, tag)) {
|
||||
if (child.id === EBML.simpleTag) simpleTags.push(child);
|
||||
else if (child.id === EBML.targets) targeted = await readTargets(walk, child);
|
||||
}
|
||||
if (targeted) continue;
|
||||
for (const simpleTag of simpleTags) await readSimpleTag(walk, simpleTag);
|
||||
}
|
||||
}
|
||||
|
||||
/** Whether the Targets name a track, chapter or attachment (ffmpeg files those tags elsewhere). */
|
||||
async function readTargets(walk, targets) {
|
||||
let targeted = false;
|
||||
for await (const field of children(walk, targets)) {
|
||||
const length = field.end - field.dataStart;
|
||||
if ((field.id === EBML.trackUid || field.id === EBML.chapterUid || field.id === EBML.attachmentUid) && length <= 8) {
|
||||
if ((await walk.read(field.dataStart, field.end)).some((byte) => byte !== 0)) targeted = true;
|
||||
}
|
||||
}
|
||||
return targeted;
|
||||
}
|
||||
|
||||
/**
|
||||
* One SimpleTag as ffmpeg's matroska_convert_tag files it: under its plain name when
|
||||
* it has no language other than und or TagDefault is set (the "-lang" name is never
|
||||
* ours), a string-less tag deletes the name, nested SimpleTags become PARENT/CHILD
|
||||
* (never ours).
|
||||
*/
|
||||
async function readSimpleTag(walk, simpleTag) {
|
||||
let name = null;
|
||||
let language = null;
|
||||
let isDefault = 0;
|
||||
let string = null;
|
||||
for await (const field of children(walk, simpleTag)) {
|
||||
const length = field.end - field.dataStart;
|
||||
if (field.id === EBML.tagName) {
|
||||
name = length <= MAX_NAME_BYTES ? utf8Strict(untilNul(await walk.read(field.dataStart, field.end))) : "";
|
||||
} else if (field.id === EBML.tagLanguage && length <= MAX_NAME_BYTES) {
|
||||
language = latin1(untilNul(await walk.read(field.dataStart, field.end)), 0, Math.min(length, MAX_NAME_BYTES));
|
||||
} else if (field.id === EBML.tagDefault && length <= 8) {
|
||||
isDefault = readUintBytes(await walk.read(field.dataStart, field.end), 0, length) ?? 0;
|
||||
} else if (field.id === EBML.tagString) {
|
||||
string = field;
|
||||
}
|
||||
}
|
||||
if (name === null || !walk.known(name)) return;
|
||||
const lang = language && language !== "und" ? language : null;
|
||||
if (isDefault === 0 && lang) return;
|
||||
if (string === null) walk.remove(name);
|
||||
else await walk.value(name, string.dataStart, string.end);
|
||||
}
|
||||
|
||||
/**
|
||||
* Reads the prompt carrier tags of a video File without touching its media data.
|
||||
*
|
||||
* @param {Blob} file - the chosen File (only `size` and `slice` are used)
|
||||
* @param {Object} [options]
|
||||
* @param {AbortSignal} [options.signal] - rejects the read with an AbortError
|
||||
* @param {Uint8Array} [options.prefix] - the file's first bytes, already read for classification
|
||||
* @param {Function} [options.yieldControl] - awaited about every 8 ms of work
|
||||
* @param {string} [options.kind] - the classifier's kind, when the prefix was not kept
|
||||
* @returns {Promise<{status:'complete', container:'iso-bmff'|'matroska', format:string|null, tags:Object<string,string>, limited:string[]}
|
||||
* | {status:'unreadable', reason:'invalid-container'|'read-failure'}
|
||||
* | {status:'limited', reason:'metadata-limit'|'chunk-limit'}
|
||||
* | {status:'unsupported', reason:'video-container-unsupported'}>}
|
||||
*/
|
||||
export async function readVideoTags(file, options = {}) {
|
||||
const { signal, yieldControl = defaultYield } = options;
|
||||
try {
|
||||
throwIfAborted(signal);
|
||||
if (!file || !Number.isSafeInteger(file.size) || typeof file.slice !== "function") throw readFailure("Not a readable file");
|
||||
let prefix = toBytes(options.prefix);
|
||||
if (!prefix && options.kind === undefined) {
|
||||
prefix = await createSliceReader(file, null, signal).read(0, Math.min(PREFIX_BYTES, file.size));
|
||||
}
|
||||
const container = options.kind ?? detectKind(prefix);
|
||||
if (container === "avi" || container === "asf") return { status: "unsupported", reason: "video-container-unsupported" };
|
||||
if (container !== "iso-bmff" && container !== "matroska") throw invalid("Not a supported video container");
|
||||
const walk = createWalk(createSliceReader(file, prefix, signal), signal, createPacer(yieldControl, signal));
|
||||
const format = container === "iso-bmff" ? await readIsoBmff(walk, file.size) : await readMatroska(walk, file.size);
|
||||
return { status: "complete", container, format, tags: walk.tags, limited: walk.limited };
|
||||
} catch (error) {
|
||||
if (error && error.name === "AbortError") throw error;
|
||||
if (error instanceof VideoReadError) return { status: error.status, reason: error.reason };
|
||||
return { status: "unreadable", reason: "read-failure" };
|
||||
}
|
||||
}
|
||||
|
||||
// ------------------------------------------------- Python json port (loads/dumps)
|
||||
|
||||
export class PythonJsonError extends Error {
|
||||
constructor(message, recursion = false) {
|
||||
super(message);
|
||||
this.name = "PythonJsonError";
|
||||
this.recursion = recursion;
|
||||
}
|
||||
}
|
||||
|
||||
/** A JSON number as Python keeps it: the lexeme, and whether it is a float. */
|
||||
class PyNumber {
|
||||
constructor(text, isFloat) {
|
||||
this.text = text;
|
||||
this.isFloat = isFloat;
|
||||
}
|
||||
}
|
||||
|
||||
const NUMBER = /-?(?:0|[1-9]\d*)(\.\d+)?([eE][-+]?\d+)?/y;
|
||||
const STRING_STOP = /["\\\u0000-\u001f]/g;
|
||||
const WHITESPACE = /[ \t\n\r]*/y;
|
||||
const ESCAPES = { '"': '"', "\\": "\\", "/": "/", b: "\b", f: "\f", n: "\n", r: "\r", t: "\t" };
|
||||
const LITERALS = [["true", true], ["false", false], ["null", null], ["NaN", new PyNumber("NaN", true)], ["Infinity", new PyNumber("Infinity", true)], ["-Infinity", new PyNumber("-Infinity", true)]];
|
||||
|
||||
/**
|
||||
* Python's ``json.loads``: objects become Maps (insertion order, last duplicate wins in
|
||||
* the first position), numbers keep their lexeme, NaN/Infinity are accepted, raw
|
||||
* control characters and trailing data are rejected.
|
||||
*/
|
||||
export function pythonJsonLoads(text) {
|
||||
if (typeof text !== "string") throw new PythonJsonError("Not a string");
|
||||
let pos = 0;
|
||||
const fail = (message) => new PythonJsonError(`${message} at ${pos}`);
|
||||
const skip = () => {
|
||||
WHITESPACE.lastIndex = pos;
|
||||
pos = WHITESPACE.exec(text)[0].length + pos;
|
||||
};
|
||||
function string() {
|
||||
pos += 1;
|
||||
let out = "";
|
||||
for (;;) {
|
||||
STRING_STOP.lastIndex = pos;
|
||||
const stop = STRING_STOP.exec(text);
|
||||
if (!stop) throw fail("Unterminated string");
|
||||
out += text.slice(pos, stop.index);
|
||||
pos = stop.index + 1;
|
||||
const char = stop[0];
|
||||
if (char === '"') return out;
|
||||
if (char !== "\\") throw fail("Invalid control character");
|
||||
const escape = text[pos];
|
||||
if (escape === "u") {
|
||||
const hex = text.slice(pos + 1, pos + 5);
|
||||
if (!/^[0-9a-fA-F]{4}$/.test(hex)) throw fail("Invalid \\uXXXX escape");
|
||||
pos += 5;
|
||||
let code = parseInt(hex, 16);
|
||||
if (code >= 0xd800 && code <= 0xdbff && text[pos] === "\\" && text[pos + 1] === "u") {
|
||||
const low = text.slice(pos + 2, pos + 6);
|
||||
if (/^[0-9a-fA-F]{4}$/.test(low)) {
|
||||
const lowCode = parseInt(low, 16);
|
||||
if (lowCode >= 0xdc00 && lowCode <= 0xdfff) {
|
||||
out += String.fromCharCode(code, lowCode);
|
||||
pos += 6;
|
||||
continue;
|
||||
}
|
||||
}
|
||||
}
|
||||
out += String.fromCharCode(code);
|
||||
} else if (escape in ESCAPES) {
|
||||
out += ESCAPES[escape];
|
||||
pos += 1;
|
||||
} else throw fail("Invalid \\escape");
|
||||
}
|
||||
}
|
||||
function value(depth) {
|
||||
if (depth > JSON_MAX_DEPTH) throw new PythonJsonError("Maximum recursion depth exceeded", true);
|
||||
const char = text[pos];
|
||||
if (char === '"') return string();
|
||||
if (char === "{") {
|
||||
pos += 1;
|
||||
const map = new Map();
|
||||
skip();
|
||||
if (text[pos] === "}") {
|
||||
pos += 1;
|
||||
return map;
|
||||
}
|
||||
for (;;) {
|
||||
if (text[pos] !== '"') throw fail("Expecting property name enclosed in double quotes");
|
||||
const key = string();
|
||||
skip();
|
||||
if (text[pos] !== ":") throw fail("Expecting ':' delimiter");
|
||||
pos += 1;
|
||||
skip();
|
||||
map.set(key, value(depth + 1));
|
||||
skip();
|
||||
if (text[pos] === "}") {
|
||||
pos += 1;
|
||||
return map;
|
||||
}
|
||||
if (text[pos] !== ",") throw fail("Expecting ',' delimiter");
|
||||
pos += 1;
|
||||
skip();
|
||||
}
|
||||
}
|
||||
if (char === "[") {
|
||||
pos += 1;
|
||||
const list = [];
|
||||
skip();
|
||||
if (text[pos] === "]") {
|
||||
pos += 1;
|
||||
return list;
|
||||
}
|
||||
for (;;) {
|
||||
list.push(value(depth + 1));
|
||||
skip();
|
||||
if (text[pos] === "]") {
|
||||
pos += 1;
|
||||
return list;
|
||||
}
|
||||
if (text[pos] !== ",") throw fail("Expecting ',' delimiter");
|
||||
pos += 1;
|
||||
skip();
|
||||
}
|
||||
}
|
||||
for (const [literal, result] of LITERALS) {
|
||||
if (text.startsWith(literal, pos)) {
|
||||
pos += literal.length;
|
||||
return result;
|
||||
}
|
||||
}
|
||||
NUMBER.lastIndex = pos;
|
||||
const number = NUMBER.exec(text);
|
||||
if (!number || number.index !== pos || !number[0]) throw fail("Expecting value");
|
||||
pos += number[0].length;
|
||||
return new PyNumber(number[0], Boolean(number[1] || number[2]));
|
||||
}
|
||||
skip();
|
||||
const result = value(0);
|
||||
skip();
|
||||
if (pos !== text.length) throw fail("Extra data");
|
||||
return result;
|
||||
}
|
||||
|
||||
/** Python's ``repr(float)``: shortest round-trip digits, exponent when decpt <= -4 or > 16. */
|
||||
function pythonFloatRepr(number) {
|
||||
if (Number.isNaN(number)) return "NaN";
|
||||
if (number === Infinity) return "Infinity";
|
||||
if (number === -Infinity) return "-Infinity";
|
||||
if (number === 0) return Object.is(number, -0) ? "-0.0" : "0.0";
|
||||
const sign = number < 0 ? "-" : "";
|
||||
const [mantissa, exponent] = Math.abs(number).toExponential().split("e");
|
||||
const digits = mantissa.replace(".", "");
|
||||
const decpt = Number(exponent) + 1;
|
||||
if (decpt <= -4 || decpt > 16) {
|
||||
const shown = digits.length > 1 ? `${digits[0]}.${digits.slice(1)}` : digits;
|
||||
const power = decpt - 1;
|
||||
return `${sign}${shown}e${power < 0 ? "-" : "+"}${String(Math.abs(power)).padStart(2, "0")}`;
|
||||
}
|
||||
if (decpt <= 0) return `${sign}0.${"0".repeat(-decpt)}${digits}`;
|
||||
if (decpt >= digits.length) return `${sign}${digits}${"0".repeat(decpt - digits.length)}.0`;
|
||||
return `${sign}${digits.slice(0, decpt)}.${digits.slice(decpt)}`;
|
||||
}
|
||||
|
||||
/** Python's ``json.dumps`` default ``ensure_ascii`` string escaping. */
|
||||
function pythonEscape(text) {
|
||||
let out = '"';
|
||||
for (let index = 0; index < text.length; index += 1) {
|
||||
const char = text[index];
|
||||
const code = text.charCodeAt(index);
|
||||
if (char === '"') out += '\\"';
|
||||
else if (char === "\\") out += "\\\\";
|
||||
else if (char === "\n") out += "\\n";
|
||||
else if (char === "\r") out += "\\r";
|
||||
else if (char === "\t") out += "\\t";
|
||||
else if (char === "\b") out += "\\b";
|
||||
else if (char === "\f") out += "\\f";
|
||||
else if (code < 0x20 || code > 0x7e) out += `\\u${code.toString(16).padStart(4, "0")}`;
|
||||
else out += char;
|
||||
}
|
||||
return `${out}"`;
|
||||
}
|
||||
|
||||
/** Python's ``json.dumps(value)`` with default separators and ``ensure_ascii``, on a loads() value. */
|
||||
export function pythonJsonDumps(value) {
|
||||
if (value === null) return "null";
|
||||
if (value === true) return "true";
|
||||
if (value === false) return "false";
|
||||
if (typeof value === "string") return pythonEscape(value);
|
||||
if (value instanceof PyNumber) {
|
||||
if (!value.isFloat) return value.text === "-0" ? "0" : value.text;
|
||||
return pythonFloatRepr(Number(value.text));
|
||||
}
|
||||
if (Array.isArray(value)) return `[${value.map(pythonJsonDumps).join(", ")}]`;
|
||||
if (value instanceof Map) return `{${[...value].map(([key, item]) => `${pythonEscape(key)}: ${pythonJsonDumps(item)}`).join(", ")}}`;
|
||||
throw new PythonJsonError("Not a JSON value");
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------ carrier selection
|
||||
|
||||
/** utils/video_metadata._record: (text or null, record format or null). */
|
||||
function record(value, key) {
|
||||
if (key === "workflow" && !(typeof value === "string" || value instanceof Map)) return [null, "invalid"];
|
||||
if (typeof value === "string") return [value, "original-text"];
|
||||
if (value instanceof Map) return [pythonJsonDumps(value), "reserialized-json"];
|
||||
return [null, null];
|
||||
}
|
||||
|
||||
/**
|
||||
* The one container carrier of prompt metadata, with the server's record budgets
|
||||
* applied (utils/video_metadata.select_video_carrier, then the metadata loop of
|
||||
* utils/prompt_inspector.read_inspection_file).
|
||||
*
|
||||
* @param {Object<string,string>} tags - container tags as ffprobe would report them
|
||||
* @param {Object} [options]
|
||||
* @param {string[]} [options.limited] - records the reader left out (`['workflow']`)
|
||||
* @returns {{status:'complete', records:Object<string,string>, record_formats:Object<string,string>, carrierText:string|null, limited:string[]}
|
||||
* | {status:'unreadable', reason:'duplicate-metadata'|'invalid-metadata'}
|
||||
* | {status:'limited', reason:'metadata-limit'}}
|
||||
*/
|
||||
export function selectVideoCarrier(tags, options = {}) {
|
||||
let limitedRecords = new Set(options.limited || []);
|
||||
const folded = new Map();
|
||||
const entries = tags && typeof tags === "object" && !Array.isArray(tags) ? Object.entries(tags) : [];
|
||||
for (const [key, value] of entries) {
|
||||
if (ENVELOPE_NAMES.has(key)) {
|
||||
folded.set(key, value);
|
||||
continue;
|
||||
}
|
||||
const name = key.toLowerCase();
|
||||
if (!PER_KEY_NAMES.has(name)) continue;
|
||||
if (folded.has(name) && folded.get(name) !== value) return { status: "unreadable", reason: "duplicate-metadata" };
|
||||
folded.set(name, value);
|
||||
}
|
||||
const candidates = [];
|
||||
const perKey = new Map(METADATA_KEYS.filter((key) => folded.has(key)).map((key) => [key, folded.get(key)]));
|
||||
if (perKey.size) candidates.push([perKey, null]);
|
||||
for (const tag of ENVELOPE_KEYS) {
|
||||
const raw = folded.get(tag);
|
||||
if (typeof raw !== "string") continue;
|
||||
let envelope;
|
||||
try {
|
||||
envelope = pythonJsonLoads(raw);
|
||||
} catch (error) {
|
||||
if (error instanceof PythonJsonError && error.recursion) return { status: "unreadable", reason: "invalid-metadata" };
|
||||
continue;
|
||||
}
|
||||
if (envelope instanceof Map) candidates.push([envelope, raw]);
|
||||
}
|
||||
const selected =
|
||||
candidates.find(([fields]) => ROLE_KEYS.some((key) => fields.has(key))) ?? candidates.find(([fields]) => fields.has("workflow")) ?? null;
|
||||
const formats = {};
|
||||
if (selected === null) {
|
||||
for (const key of limitedRecords) formats[key] = "limited";
|
||||
return { status: "complete", records: {}, record_formats: formats, carrierText: null, limited: [...limitedRecords] };
|
||||
}
|
||||
const [fields, carrierText] = selected;
|
||||
// An envelope carrier is whole; a left-out per-key workflow belonged to the carrier that lost.
|
||||
if (carrierText !== null) limitedRecords = new Set();
|
||||
const metadata = {};
|
||||
for (const key of METADATA_KEYS) {
|
||||
if (!fields.has(key)) continue;
|
||||
const [text, format] = record(fields.get(key), key);
|
||||
if (format === "invalid") {
|
||||
formats[key] = format;
|
||||
continue;
|
||||
}
|
||||
metadata[key] = text;
|
||||
if (format !== null) formats[key] = format;
|
||||
}
|
||||
const records = {};
|
||||
let roleSize = 0;
|
||||
for (const key of METADATA_KEYS) {
|
||||
if (!(key in metadata)) continue;
|
||||
const value = metadata[key];
|
||||
if (typeof value !== "string") return { status: "unreadable", reason: "invalid-metadata" };
|
||||
const size = utf8Length(value);
|
||||
if (key === "workflow") {
|
||||
if (size > WORKFLOW_METADATA_BYTES) {
|
||||
limitedRecords.add(key);
|
||||
continue;
|
||||
}
|
||||
} else {
|
||||
roleSize += size;
|
||||
if (roleSize > ROLE_METADATA_BYTES) return { status: "limited", reason: "metadata-limit" };
|
||||
}
|
||||
records[key] = value;
|
||||
if (!(key in formats)) formats[key] = "original-text";
|
||||
}
|
||||
for (const key of limitedRecords) formats[key] = "limited";
|
||||
return { status: "complete", records, record_formats: formats, carrierText, limited: [...limitedRecords] };
|
||||
}
|
||||
Reference in New Issue
Block a user