fix(inspect): read video tags as ffprobe reports them
Same bytes now give the server's record in five shapes where the local reader was stricter or kept different tags (PR #277 final review I1, I2, M1, M2, M4): - Invalid UTF-8 in a tag value is replaced as ffprobe's default string_validation=replace does (av_utf8_decode, one U+FFFD per failing decode step), instead of failing the read. Key names and TagNames that do not decode are no carrier and are skipped. - An ilst item whose keys index is 0 or out of range is skipped, as ffmpeg does, instead of failing the read. - Only the first keys box is read, as ffmpeg does. - A first per-key workflow over 4 MiB keeps its place in the carrier choice as an unread marker, so the server's carrier (per-key, workflow limited) is chosen; when the server's first probe would overflow, the re-probe leaves it out and its own bound applies. - An envelope integer of more than 4300 digits fails the Python json port, so that envelope is skipped as CPython's json.loads skips it. Each shape was checked against ffprobe 9.0.2 and read_inspection_file; the real-ffprobe parity test now builds them, and a 600-case random-byte comparison of the replacement against ffprobe matched.
This commit is contained in:
Vito Sansevero
@@ -254,10 +254,6 @@ test("containers fail closed", async () => {
|
||||
["64-bit size below its header", v.concat(v.ftyp(), v.box("mdat", v.opts({ largesize: 12 })), v.moov(v.mdtaUdta({ prompt: GRAPH })))],
|
||||
["keys count larger than the entries", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"], { count: 3 }), v.ilst(v.item(1, v.data(GRAPH))))))],
|
||||
["keys entry overrunning the box", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.box("keys", v.u32(0), v.u32(1), v.u32(90), v.latin1("mdta"), v.utf8("prompt")), v.ilst(v.item(1, v.data(GRAPH))))))],
|
||||
["ilst index 0", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(0, v.data(GRAPH))))))],
|
||||
["ilst index out of range", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(2, v.data(GRAPH))))))],
|
||||
["invalid UTF-8 in a tag value", v.perKeyMp4({ prompt: [0x7b, 0xff, 0xfe, 0x7d] })],
|
||||
["invalid UTF-8 in a key name", v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys([Uint8Array.from([0xc3, 0x28])]), v.ilst(v.item(1, v.data(GRAPH))))))],
|
||||
["no moov box", v.concat(v.ftyp(), v.mdat(), v.free(32))],
|
||||
["not a container at all", v.utf8("#EXTM3U\n#EXT-X-VERSION:3\n".padEnd(64, "x"))],
|
||||
["bad VINT: zero ID byte", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(20, { width: 8 }), [0x00, 0x81, 0x00], new Uint8Array(17))],
|
||||
@@ -266,8 +262,6 @@ test("containers fail closed", async () => {
|
||||
["element past the end of the file", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(5000, { width: 8 }), v.globalTags({ PROMPT: GRAPH }))],
|
||||
["unknown DocType", v.perKeyMkv({ PROMPT: GRAPH }, { docType: "avi" })],
|
||||
["EBML header without a DocType", v.concat(v.el(ID.EBML, v.el(ID.EBMLVersion, [1])), v.segment([v.globalTags({ PROMPT: GRAPH })]))],
|
||||
["invalid UTF-8 in a TagString", v.mkv([v.tagsEl(v.tag([v.simpleTag("PROMPT", [0xff, 0xfe])]))])],
|
||||
["invalid UTF-8 in a TagName", v.mkv([v.tagsEl(v.tag([v.simpleTag([0xff], GRAPH)]))])],
|
||||
["Matroska bytes after a stray size", v.concat(v.ebmlHeader(), v.idBytes(ID.Segment), v.vint(3, { width: 8 }), [0xec, 0x82, 0x00])],
|
||||
];
|
||||
for (const [name, bytes] of rows) {
|
||||
@@ -288,6 +282,72 @@ test("containers fail closed", async () => {
|
||||
assert.deepEqual(await readVideoTags({}), { status: "unreadable", reason: "read-failure" });
|
||||
});
|
||||
|
||||
// ffprobe's string_validation=replace (av_utf8_decode): one U+FFFD per failing
|
||||
// decode step. A bad continuation or a truncated sequence fails one byte at a time;
|
||||
// an overlong form, a surrogate, a code above U+10FFFF, U+FFFE/U+FFFF and 5/6-byte
|
||||
// forms fail as one whole sequence. Each row was read through ffprobe 9.0.2.
|
||||
const FFPROBE_REPLACEMENTS = [
|
||||
["one Latin-1 byte", [0x63, 0x61, 0x66, 0xe9], "caf�"],
|
||||
["truncated 4-byte sequence", [0x61, 0xf0, 0x9f, 0x22, 0x62], 'a��"b'],
|
||||
["overlong 2-byte form", [0x61, 0xc0, 0xaf, 0x62], "a�b"],
|
||||
["C1 overlong lead", [0x61, 0xc1, 0xbf, 0x62], "a�b"],
|
||||
["overlong 3-byte form", [0x61, 0xe0, 0x80, 0x80, 0x62], "a�b"],
|
||||
["overlong 4-byte form", [0x61, 0xf0, 0x80, 0x80, 0x80, 0x62], "a�b"],
|
||||
["UTF-16 surrogate", [0x61, 0xed, 0xa0, 0x80, 0x62], "a�b"],
|
||||
["U+FFFE", [0x61, 0xef, 0xbf, 0xbe, 0x62], "a�b"],
|
||||
["U+FFFF", [0x61, 0xef, 0xbf, 0xbf, 0x62], "a�b"],
|
||||
["U+FDD0 is kept", [0x61, 0xef, 0xb7, 0x90, 0x62], "ab"],
|
||||
["5-byte form", [0x61, 0xf8, 0x88, 0x80, 0x80, 0x80, 0x62], "a�b"],
|
||||
["6-byte form", [0x61, 0xfc, 0x84, 0x80, 0x80, 0x80, 0x80, 0x62], "a�b"],
|
||||
["truncated 5-byte form", [0x61, 0xf8, 0x88, 0x62], "a��b"],
|
||||
["above U+10FFFF", [0x61, 0xf4, 0x90, 0x80, 0x80, 0x62], "a�b"],
|
||||
["0xFE", [0x61, 0xfe, 0x62], "a�b"],
|
||||
["0xFF", [0x61, 0xff, 0x62], "a�b"],
|
||||
["0xFE 0xFF", [0xfe, 0xff], "��"],
|
||||
["lone continuation byte", [0x61, 0x80, 0x62], "a�b"],
|
||||
["truncated at the end", [0x61, 0xe2, 0x82], "a��"],
|
||||
["lead byte before a whole sequence", [0x61, 0xc3, 0xc3, 0xa9, 0x62], "a�éb"],
|
||||
["valid emoji", [0x61, 0xf0, 0x9f, 0x98, 0x80, 0x62], "a\u{1f600}b"],
|
||||
["control characters", [0x61, 0x01, 0x1f, 0x62], "a\u0001\u001fb"],
|
||||
];
|
||||
|
||||
test("invalid UTF-8 reads as ffprobe replaces it; names that do not decode are not carriers", async () => {
|
||||
for (const [name, bytes, expected] of FFPROBE_REPLACEMENTS) {
|
||||
assert.deepEqual((await read(v.perKeyMp4({ prompt: bytes }))).tags, { prompt: expected }, name);
|
||||
assert.deepEqual((await read(v.perKeyMkv({ PROMPT: bytes }))).tags, { PROMPT: expected }, `${name} (Matroska)`);
|
||||
}
|
||||
// A long value with one bad byte keeps every other byte.
|
||||
const long = Buffer.concat([Buffer.from('{"a":"'), Buffer.alloc(100000, 0x78), Buffer.from([0xe9]), Buffer.from('"}')]);
|
||||
assert.equal((await read(v.perKeyMp4({ workflow: long }))).tags.workflow, `{"a":"${"x".repeat(100000)}�"}`);
|
||||
assert.deepEqual((await read(v.perKeyMp4({ prompt: [0x7b, 0xff, 0xfe, 0x7d] }))).tags, { prompt: "{��}" });
|
||||
// A keys name with a bad byte is reported as "prom�pt", which is no carrier.
|
||||
const badKey = v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys([Uint8Array.from([0x70, 0x72, 0x6f, 0x6d, 0xff, 0x70, 0x74]), "prompt"]), v.ilst(v.item(1, v.data("junk")), v.item(2, v.data(GRAPH))))));
|
||||
assert.deepEqual(await read(badKey), complete({ prompt: GRAPH }, { container: "iso-bmff", format: "isom" }));
|
||||
const onlyBadKey = v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys([Uint8Array.from([0xc3, 0x28])]), v.ilst(v.item(1, v.data(GRAPH))))));
|
||||
assert.deepEqual((await read(onlyBadKey)).tags, {});
|
||||
assert.deepEqual((await read(v.mkv([v.tagsEl(v.tag([v.simpleTag([0xff], GRAPH), v.simpleTag("WORKFLOW", WORKFLOW)]))]))).tags, { WORKFLOW: WORKFLOW });
|
||||
assert.deepEqual((await read(v.mkv([v.tagsEl(v.tag([v.simpleTag("PROMPT", [0xff, 0xfe])]))]))).tags, { PROMPT: "��" });
|
||||
});
|
||||
|
||||
test("ilst items with index 0 or past the keys are skipped; the first keys box wins", async () => {
|
||||
const wf = '{"nodes":[]}';
|
||||
const stray = v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt", "workflow"]), v.ilst(v.item(1, v.data(GRAPH)), v.item(5, v.data("junk")), v.item(2, v.data(wf))))));
|
||||
assert.deepEqual((await read(stray)).tags, { prompt: GRAPH, workflow: wf });
|
||||
const zero = v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt", "workflow"]), v.ilst(v.item(0, v.data("junk")), v.item(1, v.data(GRAPH)), v.item(2, v.data(wf))))));
|
||||
assert.deepEqual((await read(zero)).tags, { prompt: GRAPH, workflow: wf });
|
||||
assert.deepEqual((await read(v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.ilst(v.item(2, v.data(GRAPH)))))))).tags, {});
|
||||
// ffmpeg reads one keys box: a second one (even malformed) addresses nothing.
|
||||
const twoKeys = v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.keys(["workflow"]), v.ilst(v.item(1, v.data(GRAPH))))));
|
||||
assert.deepEqual((await read(twoKeys)).tags, { prompt: GRAPH });
|
||||
const brokenSecond = v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["prompt"]), v.keys(["workflow"], { count: 9 }), v.ilst(v.item(1, v.data(GRAPH))))));
|
||||
assert.deepEqual((await read(brokenSecond)).tags, { prompt: GRAPH });
|
||||
const emptyFirst = v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys([]), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH))))));
|
||||
assert.deepEqual((await read(emptyFirst)).tags, {});
|
||||
// A keys box too short for its count field is not read, so the next one is.
|
||||
const shortFirst = v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.fullBox("keys"), v.keys(["prompt"]), v.ilst(v.item(1, v.data(GRAPH))))));
|
||||
assert.deepEqual((await read(shortFirst)).tags, { prompt: GRAPH });
|
||||
});
|
||||
|
||||
test("budgets: roles share 2 MiB, an oversized workflow is left unread, duplicates and the total are bounded", async () => {
|
||||
const limited = { status: "limited", reason: "metadata-limit" };
|
||||
// Role tags over 2 MiB together stop the read (the server fails the whole read).
|
||||
@@ -299,10 +359,11 @@ test("budgets: roles share 2 MiB, an oversized workflow is left unread, duplicat
|
||||
// A per-key workflow over 4 MiB is not sliced; the result says the workflow is limited.
|
||||
const big = v.recordingBlob(v.perKeyMp4({ prompt: GRAPH, workflow: fill(0x7b, WORKFLOW_BUDGET + 1) }));
|
||||
const over = await readVideoTags(big);
|
||||
assert.deepEqual(over, { status: "complete", container: "iso-bmff", format: "isom", tags: { prompt: GRAPH }, limited: ["workflow"] });
|
||||
// It stays in the tag map as a marker, so the carrier choice still sees a per-key workflow.
|
||||
assert.deepEqual(over, { status: "complete", container: "iso-bmff", format: "isom", tags: { prompt: GRAPH, workflow: m.OVERSIZED_WORKFLOW }, limited: ["workflow"] });
|
||||
assert.ok(big.ranges.every(([start, end]) => end - start < WORKFLOW_BUDGET), "the oversized workflow was never sliced");
|
||||
const bigMkv = await read(v.perKeyMkv({ WORKFLOW: fill(0x7b, WORKFLOW_BUDGET + 1), PROMPT: GRAPH }));
|
||||
assert.deepEqual([bigMkv.limited, Object.keys(bigMkv.tags)], [["workflow"], ["PROMPT"]]);
|
||||
assert.deepEqual([bigMkv.limited, Object.keys(bigMkv.tags)], [["workflow"], ["WORKFLOW", "PROMPT"]]);
|
||||
const atWorkflow = await read(v.perKeyMp4({ workflow: fill(0x7b, WORKFLOW_BUDGET) }));
|
||||
assert.deepEqual([atWorkflow.limited, atWorkflow.tags.workflow.length], [[], WORKFLOW_BUDGET]);
|
||||
// Fold-duplicates draw from the duplicate budgets (2x each class), then stop.
|
||||
@@ -320,6 +381,39 @@ test("budgets: roles share 2 MiB, an oversized workflow is left unread, duplicat
|
||||
assert.equal(m.VIDEO_TAG_READ_BYTES, TAG_READ_BYTES);
|
||||
});
|
||||
|
||||
test("an oversized per-key workflow keeps its place in the carrier choice, as in the server's first probe", async () => {
|
||||
const big = fill(0x7b, WORKFLOW_BUDGET + 1);
|
||||
const envelope = '{"workflow": {"nodes": [1]}}';
|
||||
const beside = (comment) => v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["workflow"]), v.ilst(v.item(1, v.data(big)), v.legacyItem("©cmt", v.data(comment))))));
|
||||
const read1 = await read(beside(envelope));
|
||||
assert.deepEqual([read1.tags, read1.limited], [{ workflow: m.OVERSIZED_WORKFLOW, comment: envelope }, ["workflow"]]);
|
||||
// The server picks the per-key carrier (no role on either side) and marks the workflow limited.
|
||||
assert.deepEqual(selectVideoCarrier(read1.tags, { limited: read1.limited }), { status: "complete", records: {}, record_formats: { workflow: "limited" }, carrierText: null, limited: ["workflow"] });
|
||||
// A role envelope still wins and clears it.
|
||||
const roleEnvelope = '{"prompt": "{}", "workflow": {"a": 1}}';
|
||||
const read2 = await read(beside(roleEnvelope));
|
||||
assert.deepEqual(selectVideoCarrier(read2.tags, { limited: read2.limited }), { status: "complete", records: { prompt: "{}", workflow: '{"a": 1}' }, record_formats: { prompt: "original-text", workflow: "reserialized-json" }, carrierText: roleEnvelope, limited: [] });
|
||||
// ISO-BMFF joins a repeat onto the oversized value, which stays over budget and unread.
|
||||
const repeat = (extra) => v.mp4(v.udta(v.meta(v.hdlr("mdta"), v.keys(["workflow", "prompt"]), v.ilst(v.item(1, v.data(big)), v.item(1, v.data(extra)), v.item(2, v.data(GRAPH))))));
|
||||
const repeatedBlob = v.recordingBlob(repeat(WORKFLOW));
|
||||
const repeated = await readVideoTags(repeatedBlob);
|
||||
assert.deepEqual([repeated.tags, repeated.limited], [{ workflow: m.OVERSIZED_WORKFLOW, prompt: GRAPH }, ["workflow"]]);
|
||||
assert.ok(repeatedBlob.ranges.every(([start, end]) => end - start < 1024), "neither workflow part was sliced");
|
||||
// When the first probe would overflow its bound, the server re-probes without the
|
||||
// per-key workflow: then it takes no part in the choice ...
|
||||
const dropped = await read(repeat(fill(0x41, TAG_READ_BYTES - WORKFLOW_BUDGET)));
|
||||
assert.deepEqual([dropped.tags, dropped.limited], [{ prompt: GRAPH }, ["workflow"]]);
|
||||
// ... and the re-probe's own bound (twice the role budget plus 64 KiB) applies to the rest.
|
||||
const wide = '{"workflow": {"n": "' + "x".repeat(TAG_READ_BYTES - WORKFLOW_BUDGET) + '"}}';
|
||||
assert.deepEqual(await read(beside(wide)), { status: "limited", reason: "metadata-limit" });
|
||||
assert.equal(m.VIDEO_REPROBE_BYTES, 2 * ROLE_BUDGET + 64 * 1024);
|
||||
// Matroska keeps the last value of a name, and a string-less SimpleTag deletes it.
|
||||
const replaced = await read(v.mkv([v.tagsEl(v.tag([v.simpleTag("WORKFLOW", big), v.simpleTag("WORKFLOW", WORKFLOW)]))]));
|
||||
assert.deepEqual([replaced.tags, replaced.limited], [{ WORKFLOW: WORKFLOW }, []]);
|
||||
const deleted = await read(v.mkv([v.tagsEl(v.tag([v.simpleTag("WORKFLOW", big), v.simpleTag("WORKFLOW", undefined), v.simpleTag("PROMPT", GRAPH)]))]));
|
||||
assert.deepEqual([deleted.tags, deleted.limited], [{ PROMPT: GRAPH }, []]);
|
||||
});
|
||||
|
||||
test("avi and asf are recognised but not read", async () => {
|
||||
const avi = v.concat(v.latin1("RIFF"), v.u32(1000), v.latin1("AVI LIST"), new Uint8Array(16));
|
||||
const asf = v.concat([0x30, 0x26, 0xb2, 0x75, 0x8e, 0x66, 0xcf, 0x11, 0xa6, 0xd9, 0x00, 0xaa, 0x00, 0x62, 0xce, 0x6c], new Uint8Array(16));
|
||||
@@ -471,6 +565,17 @@ test("pythonJsonLoads and pythonJsonDumps spell values as Python's json module d
|
||||
assert.throws(() => pythonJsonLoads(input), { name: "PythonJsonError" }, JSON.stringify(input));
|
||||
}
|
||||
assert.throws(() => pythonJsonLoads("[".repeat(2000) + "]".repeat(2000)), { name: "PythonJsonError", recursion: true });
|
||||
// CPython's int() refuses more than 4300 digits (sign excluded); floats have no such limit.
|
||||
const digits = (count) => "1" + "0".repeat(count - 1);
|
||||
assert.equal(pythonJsonDumps(pythonJsonLoads(digits(4300))), digits(4300));
|
||||
assert.equal(pythonJsonDumps(pythonJsonLoads("-" + digits(4300))), "-" + digits(4300));
|
||||
for (const input of [digits(4301), "-" + digits(4301), `[${digits(5000)}]`]) {
|
||||
assert.throws(() => pythonJsonLoads(input), (error) => error.name === "PythonJsonError" && error.recursion === false, input.slice(0, 8));
|
||||
}
|
||||
assert.equal(pythonJsonLoads(digits(4301) + ".0").isFloat, true);
|
||||
// So an envelope holding such an integer is skipped, as the server's _carriers does.
|
||||
const hugeInt = `{"workflow": {"a": ${digits(4400)}}}`;
|
||||
assert.deepEqual(selectVideoCarrier({ comment: hugeInt, description: '{"workflow": {"b": 2}}' }).carrierText, '{"workflow": {"b": 2}}');
|
||||
assert.throws(() => pythonJsonDumps(undefined), { name: "PythonJsonError" });
|
||||
assert.throws(() => pythonJsonLoads(5), { name: "PythonJsonError" });
|
||||
assert.equal(pythonJsonDumps(pythonJsonLoads("[".repeat(500) + "]".repeat(500))), "[".repeat(500) + "]".repeat(500));
|
||||
|
||||
@@ -205,6 +205,16 @@ TAG_MAP_VECTORS = [
|
||||
{"comment": ENVELOPE},
|
||||
["workflow"],
|
||||
),
|
||||
# CPython's json.loads refuses an integer of more than 4300 digits, so the
|
||||
# server skips that envelope for the next one.
|
||||
(
|
||||
"envelope_integer_over_digit_limit_skipped",
|
||||
{
|
||||
"comment": '{"workflow": {"a": 1' + "0" * 4400 + "}}",
|
||||
"description": '{"workflow": {"b": 2}}',
|
||||
},
|
||||
[],
|
||||
),
|
||||
]
|
||||
|
||||
|
||||
@@ -433,26 +443,34 @@ def _full_box(kind, *parts):
|
||||
|
||||
def _mdta_udta(tags):
|
||||
"""``moov/udta/meta`` with one mdta key per tag, laid out as ffmpeg writes it."""
|
||||
names = list(tags)
|
||||
keys = _full_box(
|
||||
b"keys",
|
||||
struct.pack(">I", len(names)),
|
||||
*(
|
||||
struct.pack(">I", 8 + len(name.encode("utf-8")))
|
||||
+ b"mdta"
|
||||
+ name.encode("utf-8")
|
||||
for name in names
|
||||
),
|
||||
)
|
||||
items = b"".join(
|
||||
_box(
|
||||
struct.pack(">I", index + 1),
|
||||
_box(b"data", struct.pack(">II", 1, 0), tags[name].encode("utf-8")),
|
||||
names = [name.encode("utf-8") for name in tags]
|
||||
values = [value.encode("utf-8") for value in tags.values()]
|
||||
return _mdta_items(names, list(enumerate(values, 1)))
|
||||
|
||||
|
||||
def _mdta_items(names, items, keys_boxes=(), legacy=()):
|
||||
"""``moov/udta/meta`` from raw bytes: mdta key ``names``, ``(index, value)``
|
||||
items, extra ``keys`` boxes after the first (each a list of names) and
|
||||
``(fourcc, value)`` legacy atoms in the same ``ilst``."""
|
||||
|
||||
def keys(entries):
|
||||
return _full_box(
|
||||
b"keys",
|
||||
struct.pack(">I", len(entries)),
|
||||
*(struct.pack(">I", 8 + len(name)) + b"mdta" + name for name in entries),
|
||||
)
|
||||
for index, name in enumerate(names)
|
||||
|
||||
def data(value):
|
||||
return _box(b"data", struct.pack(">II", 1, 0), value)
|
||||
|
||||
ilst = _box(
|
||||
b"ilst",
|
||||
*(_box(struct.pack(">I", index), data(value)) for index, value in items),
|
||||
*(_box(fourcc.encode("latin-1"), data(value)) for fourcc, value in legacy),
|
||||
)
|
||||
hdlr = _full_box(b"hdlr", b"\0\0\0\0", b"mdta", bytes(12), b"\0")
|
||||
return _box(b"udta", _full_box(b"meta", hdlr, keys, _box(b"ilst", items)))
|
||||
boxes = [keys(names), *(keys(entries) for entries in keys_boxes)]
|
||||
return _box(b"udta", _full_box(b"meta", hdlr, *boxes, ilst))
|
||||
|
||||
|
||||
def _with_udta(data, udta):
|
||||
@@ -686,7 +704,8 @@ class LocalVideoCarrierParityTests(unittest.TestCase):
|
||||
)
|
||||
limited = observed["workflow_limited"]["result"]
|
||||
self.assertEqual(limited["read"]["limited"], ["workflow"])
|
||||
self.assertNotIn("workflow", limited["read"]["tags"])
|
||||
# The unread workflow keeps its place in the carrier choice as a marker.
|
||||
self.assertEqual(limited["read"]["tags"]["workflow"], {"oversized": "workflow"})
|
||||
self.assertEqual(
|
||||
_local_shape(limited["selection"]),
|
||||
_read_shape(read_inspection_file(files["workflow_limited"])),
|
||||
@@ -703,6 +722,110 @@ class LocalVideoCarrierParityTests(unittest.TestCase):
|
||||
with self.assertRaises(MetadataReadError):
|
||||
read_inspection_file(files["roles_over_budget"])
|
||||
|
||||
@unittest.skipUnless(HAVE_FFMPEG, "ffmpeg/ffprobe not installed")
|
||||
def test_byte_level_container_shapes_read_the_same_as_the_server(self):
|
||||
"""Shapes where ffprobe is lenient: invalid UTF-8 is replaced (one U+FFFD
|
||||
per failing decode step), names that do not decode are no carrier, stray
|
||||
``ilst`` indexes are skipped, the first ``keys`` box wins, and an
|
||||
oversized per-key workflow still takes part in the carrier choice."""
|
||||
base = self.directory / "base.mp4"
|
||||
write_video(base)
|
||||
source = base.read_bytes()
|
||||
graph, workflow = GRAPH.encode(), WORKFLOW.encode()
|
||||
envelope = json.dumps({"workflow": {"nodes": [1]}}).encode()
|
||||
oversized = b'{"nodes":[' + b"0," * (5 * MiB // 2) + b"0]}"
|
||||
shapes = {
|
||||
"latin1_byte_in_workflow": _mdta_items(
|
||||
[b"prompt", b"workflow"], [(1, graph), (2, b'{"a":"caf\xe9"}')]
|
||||
),
|
||||
"undecodable_key_name": _mdta_items(
|
||||
[b"prom\xffpt", b"prompt"], [(1, b"junk"), (2, graph)]
|
||||
),
|
||||
"stray_ilst_indexes": _mdta_items(
|
||||
[b"prompt", b"workflow"],
|
||||
[(1, graph), (5, b"junk"), (0, b"junk"), (2, workflow)],
|
||||
),
|
||||
"second_keys_box": _mdta_items(
|
||||
[b"prompt"], [(1, graph)], keys_boxes=[[b"workflow"]]
|
||||
),
|
||||
"oversized_workflow_beside_envelope": _mdta_items(
|
||||
[b"workflow"], [(1, oversized)], legacy=[("\u00a9cmt", envelope)]
|
||||
),
|
||||
# A repeat joined onto it makes the first probe overflow: the server's
|
||||
# role-only re-probe reports no per-key workflow.
|
||||
"first_probe_overflows": _mdta_items(
|
||||
[b"workflow", b"prompt"],
|
||||
[(1, oversized), (1, b"A" * (8 * MiB)), (2, graph)],
|
||||
),
|
||||
}
|
||||
files = {}
|
||||
for name, udta in shapes.items():
|
||||
files[name] = self.directory / f"{name}.mp4"
|
||||
files[name].write_bytes(_with_udta(source, udta))
|
||||
# One byte of core.mkv's PROMPT TagString patched to 0xFF.
|
||||
mkv = bytearray((VIDEO_FIXTURES / "core.mkv").read_bytes())
|
||||
at = mkv.index(b'"KSampler"', mkv.index(b"PROMPT")) + 1
|
||||
mkv[at] = 0xFF
|
||||
files["matroska_invalid_byte"] = self.directory / "invalid-byte.mkv"
|
||||
files["matroska_invalid_byte"].write_bytes(bytes(mkv))
|
||||
observed = _run_oracle(
|
||||
self, [{"name": name, "path": str(path)} for name, path in files.items()]
|
||||
)
|
||||
for name, path in files.items():
|
||||
with self.subTest(shape=name):
|
||||
server = read_inspection_file(path)
|
||||
self.assertEqual(
|
||||
_local_shape(observed[name]["result"]["selection"]),
|
||||
_read_shape(server),
|
||||
)
|
||||
metadata = _metadata(server)
|
||||
self.assertEqual(
|
||||
_canonical_bytes(observed[name]["metadata"]),
|
||||
_canonical_bytes(metadata),
|
||||
)
|
||||
self.assertEqual(metadata["status"], "available")
|
||||
# The re-probe has its own bound: an 8 MiB envelope beside the oversized
|
||||
# workflow fails the server's read and stops the local one.
|
||||
wide = self.directory / "reprobe_overflows.mp4"
|
||||
wide.write_bytes(
|
||||
_with_udta(
|
||||
source,
|
||||
_mdta_items(
|
||||
[b"workflow"],
|
||||
[(1, oversized)],
|
||||
legacy=[("\u00a9cmt", b'{"n":"' + b"x" * (8 * MiB) + b'"}')],
|
||||
),
|
||||
)
|
||||
)
|
||||
local = _run_oracle(self, [{"name": "wide", "path": str(wide)}])["wide"]
|
||||
self.assertEqual(
|
||||
local["result"]["read"], {"status": "limited", "reason": "metadata-limit"}
|
||||
)
|
||||
with self.assertRaises(MetadataReadError):
|
||||
read_inspection_file(wide)
|
||||
# The server outcomes these shapes pin.
|
||||
latin1 = read_inspection_file(files["latin1_byte_in_workflow"])
|
||||
self.assertEqual(latin1.metadata["workflow"], '{"a":"caf\ufffd"}')
|
||||
self.assertEqual(
|
||||
read_inspection_file(files["stray_ilst_indexes"]).metadata,
|
||||
{"prompt": GRAPH, "workflow": WORKFLOW},
|
||||
)
|
||||
self.assertEqual(
|
||||
read_inspection_file(files["second_keys_box"]).metadata, {"prompt": GRAPH}
|
||||
)
|
||||
patched = read_inspection_file(files["matroska_invalid_byte"])
|
||||
self.assertIn('"\ufffdSampler"', patched.metadata["prompt"])
|
||||
over = read_inspection_file(files["oversized_workflow_beside_envelope"])
|
||||
self.assertEqual(
|
||||
(over.record_formats, over.video_carrier_text),
|
||||
({"workflow": "limited"}, None),
|
||||
)
|
||||
dropped = read_inspection_file(files["first_probe_overflows"])
|
||||
self.assertEqual(
|
||||
(dropped.metadata, dropped.record_formats),
|
||||
({"prompt": GRAPH}, {"prompt": "original-text", "workflow": "limited"}),
|
||||
)
|
||||
|
||||
|
||||
def _canonical(value):
|
||||
"""``value`` with every dict's keys sorted, so two orders compare as one."""
|
||||
|
||||
@@ -16,7 +16,7 @@ const FORMATS = new Set(['original-text', 'reserialized-json', 'invalid', 'limit
|
||||
const PROMPT_NODE_KEYS = new Set(['class_type', 'inputs']);
|
||||
const MAX_SAFE_DIGITS = '9007199254740991';
|
||||
// Python's int() refuses more digits than sys.get_int_max_str_digits() (4300).
|
||||
const INT_MAX_STR_DIGITS = 4300;
|
||||
export const INT_MAX_STR_DIGITS = 4300;
|
||||
// decimal's exact constructor (libmpdec, 64-bit) refuses values outside its
|
||||
// maximal context: adjusted exponent above MAX_EMAX or exponent below Etiny.
|
||||
const DECIMAL_MAX_EMAX = 999999999999999999n;
|
||||
|
||||
@@ -14,6 +14,8 @@
|
||||
// exact-case (comment, description) and the per-key names fold ASCII case in the
|
||||
// selector, exactly as the server folds them.
|
||||
|
||||
import { INT_MAX_STR_DIGITS } from "./local-projection.mjs";
|
||||
|
||||
// Role records (prompt, parameters, the pair execution record) share one budget:
|
||||
// past it the read fails. A workflow has its own budget; past it only the workflow
|
||||
// is "limited". Duplicate tags draw from separate budgets per class. The total of
|
||||
@@ -23,6 +25,11 @@ export const WORKFLOW_METADATA_BYTES = 4 * 1024 * 1024;
|
||||
export const DUPLICATE_WORKFLOW_BYTES = 2 * WORKFLOW_METADATA_BYTES;
|
||||
export const DUPLICATE_ROLE_BYTES = 2 * ROLE_METADATA_BYTES;
|
||||
export const VIDEO_TAG_READ_BYTES = 2 * (ROLE_METADATA_BYTES + WORKFLOW_METADATA_BYTES) + 64 * 1024;
|
||||
// The server's role-only re-probe bound, taken when the first probe overflows.
|
||||
export const VIDEO_REPROBE_BYTES = 2 * ROLE_METADATA_BYTES + 64 * 1024;
|
||||
// A first per-key workflow over its budget is never sliced; this marker holds its
|
||||
// place in the tag map, as the server's first probe still reports the tag.
|
||||
export const OVERSIZED_WORKFLOW = Object.freeze({ oversized: "workflow" });
|
||||
export const MAX_VIDEO_ELEMENTS = 10000;
|
||||
const MAX_HEADER_BYTES = 16; // a 64-bit ISO-BMFF box header; an EBML header is at most 12
|
||||
const MAX_NAME_BYTES = 64; // the longest carrier name is 24 bytes; longer names are never ours
|
||||
@@ -132,13 +139,60 @@ function untilNul(bytes) {
|
||||
return nul < 0 ? bytes : bytes.subarray(0, nul);
|
||||
}
|
||||
|
||||
/** Strict UTF-8, as Python decodes ffprobe's output: invalid bytes fail the read. */
|
||||
function utf8Strict(bytes) {
|
||||
try {
|
||||
return new TextDecoder("utf-8", { fatal: true, ignoreBOM: true }).decode(bytes);
|
||||
} catch (error) {
|
||||
throw invalid("Video metadata is not valid UTF-8");
|
||||
const STRICT_UTF8 = new TextDecoder("utf-8", { fatal: true, ignoreBOM: true });
|
||||
const REPLACEMENT = "\ufffd";
|
||||
// av_utf8_decode's smallest code point per continuation count (overlong below it).
|
||||
const OVERLONG_MINIMUM = [0, 0x80, 0x800, 0x10000];
|
||||
|
||||
/**
|
||||
* One av_utf8_decode step (flags 0, as ffprobe calls it) from `start`: the next
|
||||
* offset and whether the sequence is valid. A lead byte that is a continuation or
|
||||
* 0xFE/0xFF, an overlong form, a surrogate, a code above U+10FFFF, U+FFFE/U+FFFF
|
||||
* and every 5/6-byte form fail as the whole sequence; a missing or bad
|
||||
* continuation byte fails the lead byte alone.
|
||||
*/
|
||||
function ffprobeStep(bytes, start) {
|
||||
const lead = bytes[start];
|
||||
if ((lead & 0xc0) === 0x80 || lead >= 0xfe) return [start + 1, false];
|
||||
let tail = 0;
|
||||
if (lead & 0x80) while (lead & (0x40 >> tail)) tail += 1;
|
||||
let code = tail ? lead & (0x3f >> tail) : lead;
|
||||
let next = start + 1;
|
||||
for (let index = 0; index < tail; index += 1, next += 1) {
|
||||
if (next >= bytes.length || (bytes[next] & 0xc0) !== 0x80) return [start + 1, false];
|
||||
code = code * 64 + (bytes[next] & 0x3f);
|
||||
}
|
||||
const valid = tail <= 3 && code >= OVERLONG_MINIMUM[tail] && code <= 0x10ffff && !(code >= 0xd800 && code <= 0xdfff) && code !== 0xfffe && code !== 0xffff;
|
||||
return [next, valid];
|
||||
}
|
||||
|
||||
/**
|
||||
* Tag text as ffprobe prints it with its default string_validation=replace
|
||||
* (fftools validate_string): each failing av_utf8_decode step becomes one U+FFFD,
|
||||
* so Python's json.loads of ffprobe's output never meets invalid UTF-8. This
|
||||
* differs from WHATWG decoding in how many replacements a bad sequence yields.
|
||||
*/
|
||||
function ffprobeText(bytes) {
|
||||
let text = null;
|
||||
try {
|
||||
text = STRICT_UTF8.decode(bytes);
|
||||
} catch (error) {
|
||||
text = null;
|
||||
}
|
||||
if (text !== null && !/[\ufffe\uffff]/.test(text)) return text;
|
||||
const parts = [];
|
||||
let run = 0;
|
||||
for (let offset = 0; offset < bytes.length; ) {
|
||||
const [next, valid] = ffprobeStep(bytes, offset);
|
||||
if (!valid) {
|
||||
if (offset > run) parts.push(STRICT_UTF8.decode(bytes.subarray(run, offset)));
|
||||
parts.push(REPLACEMENT);
|
||||
run = next;
|
||||
}
|
||||
offset = next;
|
||||
}
|
||||
if (bytes.length > run) parts.push(STRICT_UTF8.decode(bytes.subarray(run)));
|
||||
return parts.join("");
|
||||
}
|
||||
|
||||
/** UTF-8 length as Python's ``encode("utf-8", "surrogatepass")`` counts it. */
|
||||
@@ -226,7 +280,13 @@ function createWalk(reader, signal, pacer) {
|
||||
let duplicateRole = 0;
|
||||
let duplicateWorkflow = 0;
|
||||
let total = 0;
|
||||
let workflowRead = 0; // bytes of per-key workflow tags read, which the re-probe leaves out
|
||||
let oversized = null; // {name, size}: the unread first per-key workflow
|
||||
const overLimit = () => limited("metadata-limit", "Video metadata exceeds inspection limits");
|
||||
const forgetOversized = () => {
|
||||
oversized = null;
|
||||
limitedRecords.splice(limitedRecords.indexOf("workflow"), 1);
|
||||
};
|
||||
return {
|
||||
tags,
|
||||
limited: limitedRecords,
|
||||
@@ -242,8 +302,19 @@ function createWalk(reader, signal, pacer) {
|
||||
return PER_KEY_NAMES.has(name.toLowerCase()) || ENVELOPE_NAMES.has(name);
|
||||
},
|
||||
remove(name) {
|
||||
if (oversized && oversized.name === name) forgetOversized();
|
||||
delete tags[name];
|
||||
},
|
||||
/**
|
||||
* Settles the oversized workflow as the server's probes would: when the
|
||||
* first probe fits, the tag keeps its marker; when it overflows, the
|
||||
* role-only re-probe reports no per-key workflow and has its own bound.
|
||||
*/
|
||||
finish() {
|
||||
if (!oversized || total + oversized.size <= VIDEO_TAG_READ_BYTES) return;
|
||||
for (const name of Object.keys(tags)) if (name.toLowerCase() === "workflow") delete tags[name];
|
||||
if (total - workflowRead > VIDEO_REPROBE_BYTES) throw overLimit();
|
||||
},
|
||||
/**
|
||||
* Reads and keeps the tag `name` (a known carrier, see `known`) whose value
|
||||
* spans [start, end), within the budgets. `join` appends to an existing value
|
||||
@@ -258,10 +329,21 @@ function createWalk(reader, signal, pacer) {
|
||||
seen.add(folded);
|
||||
if (folded === "workflow") {
|
||||
if (first && size > WORKFLOW_METADATA_BYTES) {
|
||||
// The server's role-only re-probe: the per-key workflow is left out.
|
||||
if (!limitedRecords.includes("workflow")) limitedRecords.push("workflow");
|
||||
oversized = { name, size };
|
||||
tags[name] = OVERSIZED_WORKFLOW;
|
||||
limitedRecords.push("workflow");
|
||||
return;
|
||||
}
|
||||
if (tags[name] === OVERSIZED_WORKFLOW) {
|
||||
// An ISO-BMFF repeat joins onto the unread value, still over budget.
|
||||
if (join !== null) {
|
||||
oversized.size += size;
|
||||
return;
|
||||
}
|
||||
// Matroska keeps a name's last value: the oversized one is gone.
|
||||
forgetOversized();
|
||||
}
|
||||
workflowRead += size;
|
||||
if (first) workflow += size;
|
||||
else duplicateWorkflow += size;
|
||||
if (duplicateWorkflow > DUPLICATE_WORKFLOW_BYTES) throw overLimit();
|
||||
@@ -273,7 +355,7 @@ function createWalk(reader, signal, pacer) {
|
||||
}
|
||||
total += size;
|
||||
if (total > VIDEO_TAG_READ_BYTES) throw overLimit();
|
||||
const value = text ?? utf8Strict(untilNul(await reader.read(start, end)));
|
||||
const value = text ?? ffprobeText(untilNul(await reader.read(start, end)));
|
||||
tags[name] = join !== null && name in tags ? `${tags[name]}${join}${value}` : value;
|
||||
},
|
||||
};
|
||||
@@ -361,9 +443,12 @@ async function readMeta(walk, meta, state) {
|
||||
}
|
||||
}
|
||||
|
||||
/** The `keys` box: 1-based mdta names; other namespaces and overlong names address nothing. */
|
||||
/**
|
||||
* The `keys` box: 1-based mdta names; other namespaces and overlong names address
|
||||
* nothing. As ffmpeg, only the first keys box with a count field is read.
|
||||
*/
|
||||
async function readKeys(walk, box, state) {
|
||||
if (box.end - box.bodyStart < 8) return;
|
||||
if (state.keys || box.end - box.bodyStart < 8) return;
|
||||
const head = await walk.read(box.bodyStart, box.bodyStart + 8);
|
||||
const count = readUint32(head, 4);
|
||||
const keys = [null];
|
||||
@@ -376,7 +461,7 @@ async function readKeys(walk, box, state) {
|
||||
const nameLength = keySize - 8;
|
||||
let name = null;
|
||||
if (latin1(entry, 4, 8) === "mdta" && nameLength <= MAX_NAME_BYTES) {
|
||||
name = utf8Strict(await walk.read(offset + 8, offset + keySize));
|
||||
name = ffprobeText(await walk.read(offset + 8, offset + keySize));
|
||||
}
|
||||
keys.push(name);
|
||||
offset += keySize;
|
||||
@@ -406,7 +491,8 @@ async function readIlst(walk, ilst, state) {
|
||||
let name = LEGACY_ATOMS.get(item.type);
|
||||
if (name === undefined && state.hdlrMdta && state.keys) {
|
||||
const index = readUint32(item.raw, 0);
|
||||
if (index === 0 || index >= state.keys.length) throw invalid("ilst index out of range");
|
||||
// ffmpeg warns that the index is out of range and skips the item.
|
||||
if (index === 0 || index >= state.keys.length) continue;
|
||||
name = state.keys[index];
|
||||
}
|
||||
if (!name || !walk.known(name)) continue;
|
||||
@@ -552,7 +638,7 @@ async function readSimpleTag(walk, simpleTag) {
|
||||
for await (const field of children(walk, simpleTag)) {
|
||||
const length = field.end - field.dataStart;
|
||||
if (field.id === EBML.tagName) {
|
||||
name = length <= MAX_NAME_BYTES ? utf8Strict(untilNul(await walk.read(field.dataStart, field.end))) : "";
|
||||
name = length <= MAX_NAME_BYTES ? ffprobeText(untilNul(await walk.read(field.dataStart, field.end))) : "";
|
||||
} else if (field.id === EBML.tagLanguage && length <= MAX_NAME_BYTES) {
|
||||
language = latin1(untilNul(await walk.read(field.dataStart, field.end)), 0, Math.min(length, MAX_NAME_BYTES));
|
||||
} else if (field.id === EBML.tagDefault && length <= 8) {
|
||||
@@ -596,6 +682,7 @@ export async function readVideoTags(file, options = {}) {
|
||||
if (container !== "iso-bmff" && container !== "matroska") throw invalid("Not a supported video container");
|
||||
const walk = createWalk(createSliceReader(file, prefix, signal), signal, createPacer(yieldControl, signal));
|
||||
const format = container === "iso-bmff" ? await readIsoBmff(walk, file.size) : await readMatroska(walk, file.size);
|
||||
walk.finish();
|
||||
return { status: "complete", container, format, tags: walk.tags, limited: walk.limited };
|
||||
} catch (error) {
|
||||
if (error && error.name === "AbortError") throw error;
|
||||
@@ -737,7 +824,10 @@ export function pythonJsonLoads(text) {
|
||||
const number = NUMBER.exec(text);
|
||||
if (!number || number.index !== pos || !number[0]) throw fail("Expecting value");
|
||||
pos += number[0].length;
|
||||
return new PyNumber(number[0], Boolean(number[1] || number[2]));
|
||||
const isFloat = Boolean(number[1] || number[2]);
|
||||
// CPython's int() refuses more digits than sys.get_int_max_str_digits().
|
||||
if (!isFloat && number[0].replace("-", "").length > INT_MAX_STR_DIGITS) throw fail("Exceeds the limit for integer string conversion");
|
||||
return new PyNumber(number[0], isFloat);
|
||||
}
|
||||
skip();
|
||||
const result = value(0);
|
||||
@@ -864,6 +954,11 @@ export function selectVideoCarrier(tags, options = {}) {
|
||||
const metadata = {};
|
||||
for (const key of METADATA_KEYS) {
|
||||
if (!fields.has(key)) continue;
|
||||
if (fields.get(key) === OVERSIZED_WORKFLOW) {
|
||||
// The reader left it unread: the server reads it, then finds it over budget.
|
||||
limitedRecords.add(key);
|
||||
continue;
|
||||
}
|
||||
const [text, format] = record(fields.get(key), key);
|
||||
if (format === "invalid") {
|
||||
formats[key] = format;
|
||||
|
||||
Reference in New Issue
Block a user