fix(inspect): one JSON depth cap for both browser parsers
py-json.mjs and local-video.mjs now refuse past the same 1,000 nested arrays or objects, counting every container as CPython's scanner does. Boundary vectors at 999, 1,000 and 1,001 for a deep prompt and a deep workflow run through the Python oracle, and TESTING.md gives the measured per-version server limits and both divergence directions.
This commit is contained in:
+24
-6
@@ -2324,15 +2324,33 @@ skip without ffmpeg). The remaining known divergences, checked against ffprobe 9
|
||||
`unreadable`); ffprobe reads such files.
|
||||
- An EBML DocType other than `matroska`/`webm` is an invalid container locally; ffmpeg
|
||||
only warns and reads it.
|
||||
- Local roles (PR L3): a prompt graph nested deeper than CPython's `json.loads`
|
||||
recursion limit (about 1,000 levels, depending on the Python version) is malformed on
|
||||
the server and parsed locally. `\d` in the A1111 settings footer is any Unicode decimal
|
||||
- Deep JSON (local roles, PR L3, and video envelopes): both browser CPython-JSON
|
||||
parsers (`py-json.mjs`, `local-video.mjs`) share one cap, `JSON_MAX_DEPTH` = 1,000
|
||||
nested arrays or objects (each counts, an empty one too); past it a record is invalid
|
||||
JSON locally. The server's `json.loads` raises RecursionError at a limit that depends
|
||||
on the Python, measured on Linux x86-64: 993 levels from a bare script on 3.10 and
|
||||
3.11 (the 1,000 recursion limit minus the caller's frames, so about 990 inside the
|
||||
server), 9,997 on 3.12 and 9,998 on 3.13 (the C recursion counter; lower on Windows
|
||||
builds), and about 52,150 on 3.14 (stack-based, varies with the thread's stack).
|
||||
The record kinds answer: a deep `prompt` graph (or `comfyassets_prompt_pairs`
|
||||
envelope) is malformed; a deep `workflow` is ignored, so the overwritten PromptManager
|
||||
check (R1) does not run and a prompt can resolve; a deep video `comment` or
|
||||
`description` envelope makes the file unreadable on both sides (the server's
|
||||
RecursionError reaches `read_inspection_file`). That gives two directions:
|
||||
- 3.10/3.11, between the server's limit and 1,000: the server refuses and the local
|
||||
page reads, so a deep prompt resolves locally while the server says malformed, and
|
||||
a deep workflow can make R1 refuse locally where the server resolves.
|
||||
- 3.12 and newer, from 1,001 up to the server's limit: the local page refuses and
|
||||
the server reads, so a deep prompt is malformed locally while the server resolves,
|
||||
and a deep workflow is ignored locally while the server can refuse it under R1.
|
||||
Past both limits they agree. `test_local_json_depth_cap_at_its_boundary` checks the
|
||||
local answer at 999, 1,000 and 1,001 for both kinds and that any server difference
|
||||
is only one of the two directions above.
|
||||
- Local roles (PR L3): `\d` in the A1111 settings footer is any Unicode decimal
|
||||
digit on both sides, but the browser's Unicode version can be newer than the server
|
||||
Python's, so a footer whose step count uses a digit added after Unicode 15 can differ.
|
||||
`str.lower()` and `toLowerCase()` differ only for characters no node class name uses.
|
||||
- Envelope JSON deeper than 1,000 levels is refused locally as Python's default
|
||||
recursion limit refuses it; the server's real limit depends on the interpreter and
|
||||
call stack. A duplicate key in an object enclosing a nest deeper than the running
|
||||
- A duplicate key in an object enclosing a nest deeper than the running
|
||||
Python's json limit is `malformed` locally and `limited` on that Python.
|
||||
- Decimal exponent limits assume CPython's C `_decimal`; on a `_pydecimal` interpreter
|
||||
the server would read the over-limit exponents the local projection refuses.
|
||||
|
||||
@@ -27,6 +27,25 @@ test('json.loads keeps Python value kinds and dict order', () => {
|
||||
assert.equal(py.pyJsonLoads(` ${'9'.repeat(4300)}\n`), BigInt('9'.repeat(4300)));
|
||||
});
|
||||
|
||||
// One depth cap for both browser CPython-JSON parsers (py-json.mjs here, local-video.mjs for video envelopes): every
|
||||
// array or object counts, an empty one too, as CPython's scanner enters recursion for each; past the cap it refuses.
|
||||
test('json.loads refuses nesting past the shared depth cap, counting every container', () => {
|
||||
const video = import('../../web/js/workspace/local-video.mjs');
|
||||
assert.equal(py.JSON_MAX_DEPTH, 1000);
|
||||
const nest = (depth, inner = '') => '['.repeat(depth) + inner + ']'.repeat(depth);
|
||||
const objects = (depth, inner) => '{"a":'.repeat(depth) + inner + '}'.repeat(depth);
|
||||
for (const text of [nest(1000), nest(1000, '1'), nest(999, '{}'), objects(1000, '1'), objects(999, '[]')]) assert.doesNotThrow(() => py.pyJsonLoads(text));
|
||||
for (const text of [nest(1001), nest(1001, '1'), nest(1000, '{}'), objects(1001, '1'), objects(1000, '[]')]) {
|
||||
assert.throws(() => py.pyJsonLoads(text), py.PyValueError);
|
||||
}
|
||||
return video.then(({ pythonJsonLoads, PythonJsonError }) => {
|
||||
for (const text of [nest(1000), nest(1000, '1'), objects(999, '[]')]) assert.doesNotThrow(() => pythonJsonLoads(text));
|
||||
for (const text of [nest(1001), nest(1001, '1'), objects(1000, '[]')]) {
|
||||
assert.throws(() => pythonJsonLoads(text), error => error instanceof PythonJsonError && error.recursion === true);
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
test('str helpers measure, slice and strip as CPython', () => {
|
||||
assert.equal(py.pyLen('a\u{1f600}\ud800'), 3);
|
||||
assert.equal(py.pySlice('\u{1f600}\u{1f600}x', 2), '\u{1f600}\u{1f600}');
|
||||
|
||||
@@ -997,6 +997,48 @@ def _threshold(build, low, high):
|
||||
return low
|
||||
|
||||
|
||||
# The nesting the browser's CPython-JSON parsers accept (py-json.mjs JSON_MAX_DEPTH).
|
||||
JSON_MAX_DEPTH = 1000
|
||||
JSON_DEPTHS = (JSON_MAX_DEPTH - 1, JSON_MAX_DEPTH, JSON_MAX_DEPTH + 1)
|
||||
|
||||
|
||||
def _nested(depth):
|
||||
"""``depth`` nested arrays."""
|
||||
return "[" * depth + "]" * depth
|
||||
|
||||
|
||||
def depth_cases():
|
||||
"""``(name, kind, depth, records)``: a deep record at the JSON depth cap.
|
||||
|
||||
A ``prompt`` holds a junk input nested to ``depth`` containers in all (the
|
||||
graph, its node and the inputs are three); parsed, both roles resolve.
|
||||
A ``workflow`` nested to ``depth`` makes two PromptManager nodes
|
||||
overwritten (R1) when it is read, and the prompt resolves when it is not.
|
||||
"""
|
||||
cases = []
|
||||
base = {"6": _encode("a fox"), "7": _encode("blurry")}
|
||||
for depth in JSON_DEPTHS:
|
||||
graph = _dump(_graph(["6", 0], ["7", 0], {**base, "9": {"class_type": "Note"}}))
|
||||
junk = '"inputs": {"junk": ' + _nested(depth - 3) + "}"
|
||||
prompt = graph.replace('"class_type": "Note"', '"class_type": "Note", ' + junk)
|
||||
cases.append((f"prompt-depth-{depth}", "prompt", depth, {"prompt": prompt}))
|
||||
overwritten = {**base, "6": _encode("shared"), "7": _encode("shared")}
|
||||
overwritten["6"]["inputs"]["search_text"] = ""
|
||||
nodes = [
|
||||
{"id": 6, "type": "PromptManager", "widgets_values": ["own text"]},
|
||||
{"id": 7, "type": "PromptManager", "widgets_values": ["other text"]},
|
||||
]
|
||||
workflow = json.dumps({"nodes": nodes, "extra": 0}).replace(
|
||||
'"extra": 0', '"extra": ' + _nested(depth - 1)
|
||||
)
|
||||
records = {
|
||||
"prompt": _dump(_graph(["6", 0], ["7", 0], overwritten)),
|
||||
"workflow": workflow,
|
||||
}
|
||||
cases.append((f"workflow-depth-{depth}", "workflow", depth, records))
|
||||
return cases
|
||||
|
||||
|
||||
def boundary_cases():
|
||||
"""Each work bound at its edge: the last value that resolves, then the next."""
|
||||
cases = {}
|
||||
|
||||
@@ -1356,6 +1356,53 @@ class LocalResolverParityTests(unittest.TestCase):
|
||||
"\U0001f600" * 64,
|
||||
)
|
||||
|
||||
def test_local_json_depth_cap_at_its_boundary(self):
|
||||
"""Both kinds of deep record at the browser's JSON depth cap (1,000).
|
||||
|
||||
Locally a record nested past the cap is invalid JSON: a prompt is
|
||||
malformed and a workflow is ignored, as the server answers when
|
||||
``json.loads`` raises RecursionError. The server's own limit depends on
|
||||
the Python (TESTING.md, "Deep JSON"): about 990 on 3.10 and 3.11, so it
|
||||
agrees past the cap there; about 10,000 on 3.12 and 3.13 and over
|
||||
50,000 on 3.14, so it agrees up to the cap there and reads deeper.
|
||||
"""
|
||||
cases = resolver_cases.depth_cases()
|
||||
observed = _run_oracle(
|
||||
self,
|
||||
[
|
||||
{"name": name, "mode": "resolve", "records": records}
|
||||
for name, _kind, _depth, records in cases
|
||||
],
|
||||
)
|
||||
newer = sys.version_info >= (3, 12)
|
||||
compared = 0
|
||||
for name, kind, depth, records in cases:
|
||||
with self.subTest(case=name):
|
||||
local = observed[name]["evidence"]["resolution"]
|
||||
within = depth <= resolver_cases.JSON_MAX_DEPTH
|
||||
if kind == "prompt":
|
||||
expected = (
|
||||
("resolved", None) if within else ("unresolved", "malformed")
|
||||
)
|
||||
else:
|
||||
# A workflow that is read finds the overwritten pair (R1).
|
||||
expected = (
|
||||
("unresolved", "malformed") if within else ("resolved", None)
|
||||
)
|
||||
self.assertEqual((local["state"], local["reason_code"]), expected)
|
||||
server = _server_evidence(records)
|
||||
if server["resolution"]["state"] == local["state"]:
|
||||
self.assertEqual(
|
||||
_canonical_bytes(observed[name]["evidence"]),
|
||||
_canonical_bytes(server),
|
||||
)
|
||||
compared += 1
|
||||
else:
|
||||
# The documented divergence only: an older server refuses
|
||||
# a little below the cap, a newer one reads past it.
|
||||
self.assertEqual(within, not newer)
|
||||
self.assertGreaterEqual(compared, 2)
|
||||
|
||||
def test_local_roles_match_fuzzed_record_sets(self):
|
||||
cases = {}
|
||||
for seed in range(4):
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
// server folds them.
|
||||
|
||||
import { INT_MAX_STR_DIGITS } from "./local-projection.mjs";
|
||||
import { JSON_MAX_DEPTH } from "./py-json.mjs";
|
||||
|
||||
// Role records (prompt, parameters, the pair execution record) share one budget:
|
||||
// past it the read fails. A workflow has its own budget; past it only the workflow
|
||||
@@ -40,7 +41,6 @@ const MAX_NAME_BYTES = 64; // the longest carrier name is 24 bytes; longer names
|
||||
export const VIDEO_READ_CEILING = MAX_VIDEO_ELEMENTS * (MAX_HEADER_BYTES + MAX_NAME_BYTES) + VIDEO_TAG_READ_BYTES;
|
||||
const PREFIX_BYTES = 32;
|
||||
const YIELD_INTERVAL_MS = 8;
|
||||
const JSON_MAX_DEPTH = 1000; // Python's default recursion limit
|
||||
|
||||
const ROLE_KEYS = ["prompt", "parameters", "comfyassets_prompt_pairs"];
|
||||
const METADATA_KEYS = ["prompt", "workflow", "parameters", "comfyassets_prompt_pairs"];
|
||||
@@ -826,8 +826,9 @@ export function pythonJsonLoads(text) {
|
||||
}
|
||||
}
|
||||
function value(depth) {
|
||||
if (depth > JSON_MAX_DEPTH) throw new PythonJsonError("Maximum recursion depth exceeded", true);
|
||||
const char = text[pos];
|
||||
// The shared cap (py-json.mjs): `depth` containers enclose this value, and it would be one more.
|
||||
if ((char === "{" || char === "[") && depth >= JSON_MAX_DEPTH) throw new PythonJsonError("Maximum recursion depth exceeded", true);
|
||||
if (char === '"') return string();
|
||||
if (char === "{") {
|
||||
pos += 1;
|
||||
|
||||
@@ -22,6 +22,12 @@ export class PyValueError extends Error {
|
||||
|
||||
// CPython's default int max str digits (PEP 651 / gh-95778).
|
||||
const INT_MAX_STR_DIGITS = 4300;
|
||||
/**
|
||||
* The nesting both browser CPython-JSON parsers accept (this one and local-video.mjs's): every array or object counts,
|
||||
* an empty one too, as CPython's scanner enters a recursion level for each. Past it `json.loads` raises RecursionError,
|
||||
* which the server treats as invalid JSON; its real limit depends on the Python (TESTING.md, "Deep JSON").
|
||||
*/
|
||||
export const JSON_MAX_DEPTH = 1000;
|
||||
const MAX_SAFE_DIGITS = '9007199254740991';
|
||||
const NUMBER = /-?(?:0|[1-9][0-9]*)(\.[0-9]+)?([eE][-+]?[0-9]+)?/y;
|
||||
const STRING_STOP = /["\\\u0000-\u001f]/g;
|
||||
@@ -118,17 +124,21 @@ export function pyJsonLoads(text) {
|
||||
}
|
||||
skip();
|
||||
const stack = [];
|
||||
const enter = () => {
|
||||
if (stack.length >= JSON_MAX_DEPTH) throw fail('Maximum JSON nesting depth exceeded');
|
||||
pos++; skip();
|
||||
};
|
||||
let result;
|
||||
for (;;) {
|
||||
let value;
|
||||
if (text[pos] === '{') {
|
||||
pos++; skip();
|
||||
enter();
|
||||
const frame = { value: new Map(), object: true, key: null };
|
||||
if (text[pos] === '}') pos++;
|
||||
else { frame.key = key(); stack.push(frame); continue; }
|
||||
value = frame.value;
|
||||
} else if (text[pos] === '[') {
|
||||
pos++; skip();
|
||||
enter();
|
||||
const frame = { value: [], object: false };
|
||||
if (text[pos] === ']') pos++;
|
||||
else { stack.push(frame); continue; }
|
||||
|
||||
Reference in New Issue
Block a user