diff --git a/TESTING.md b/TESTING.md index c46b73a..e0c56a7 100644 --- a/TESTING.md +++ b/TESTING.md @@ -194,10 +194,38 @@ The inspector does not establish historical file identity, infer run membership, edit image tags, export all EXIF, or load/execute a workflow. Saved vocabulary is labelled by its recipe or positive-prompt scope. Limited raw evidence is omitted atomically, with JSON exports unavailable and the rooted original still downloadable. -The inspection API uses one bounded read (2 MiB aggregate metadata, 8 MiB non-PNG -header, 10,000 PNG chunks, three-second ffprobe, 500 graph nodes/semantic expansions, -64 link/JSON-container levels and an 8 MiB serialized response); retry does not raise -these limits. Installing the Python inspection/source-scope changes requires a +The inspection API uses one bounded read with per-record budgets: role records +(`prompt`, `parameters`, the pair execution record) share 2 MiB and exceeding it +fails the read; the `workflow` record has its own 4 MiB and past it only the workflow +becomes record format `limited` (no raw text, graph `limited`, roles and facts +unaffected); other text chunks are skipped unread; byte-identical duplicates are +accepted within a bounded read budget, while differing duplicates (and any duplicate +role record over that budget) still fail. Other read bounds are unchanged: 8 MiB +non-PNG header, 10,000 PNG chunks and a three-second ffprobe. Resolution under +`INSPECTION_PROFILE` bounds work, not graph size: 2,000 visits per candidate chain, +20,000 in total, 64 link/JSON-container levels, 100,000 resolved characters, and a +two 20,000-node ceilings: one over the top-level entries of the API prompt, and one +over the workflow graph's UI nodes plus subgraph-definition nodes (past it the +workflow graph and its facts become limited). The serialized +response stays under 8 MiB by omitting, in order, the workflow graph, then the prompt +graph, then all raw records atomically. Retry does not raise these limits. Ingestion +(scan, live monitor, admin import) and queue-time run counting keep the default +profile, including its 500-node gate. + +`tools/inspection_corpus.py --root DIR --out JSONL [--workers N] [--baseline JSONL] +[--summary TXT] [--code TREE]` measures a media folder read-only against the #240 +invariants H1 to H8. It runs the inspection route pipeline and the workspace JS facts +of `--code` (default: its own tree), opens media only for reading, follows no +symlinks, imports no database, and refuses `--out`/`--summary` inside the root or a +code tree. Its rows hold statuses, sizes, timings and SHA-256 digests of resolved +role texts, never file names or prompt text unless `--with-paths` is given; digests +of short prompts can still be guessed, so the JSONL stays local and only the +aggregate summary goes into a PR body. Run it on the base commit first and pass that +JSONL as `--baseline` for H1 and H8 (2x the baseline p99, never below 50 ms). It +exits 1 when an invariant fails, including the `harness` line: a tool exception or +facts that were not computed (no Node.js) fail the run. Unreadable or unsupported +files are inspector outcomes, not harness errors. Never point it at a live install's folders from an +automated job. Installing the Python inspection/source-scope changes requires a maintainer ComfyUI restart. Automated tests use disposable hosts and never restart the running personal ComfyUI. diff --git a/py/api/inspection_service.py b/py/api/inspection_service.py index 5f4e0d7..0b15bd2 100644 --- a/py/api/inspection_service.py +++ b/py/api/inspection_service.py @@ -82,6 +82,8 @@ def inspect_current_file( "state": "unresolved", "sampler_id": None, "reason": "Media metadata could not be read", + "reason_code": None, + "node": None, }, } else: diff --git a/tests/browser/full-inspector.spec.mjs b/tests/browser/full-inspector.spec.mjs index a8bd232..167974d 100644 --- a/tests/browser/full-inspector.spec.mjs +++ b/tests/browser/full-inspector.spec.mjs @@ -153,6 +153,21 @@ test('full inspector downloads actual bounded evidence',async({page,request})=>{ await panel.getByText('File & provenance',{exact:true}).click();await expect(panel.getByRole('link',{name:'Download original',exact:true})).toHaveAttribute('href',/root=0&root_scope=[a-f0-9]{64}/); }); +test('full inspector labels a limited workflow record without hiding prompt facts',async({page,request})=>{ + // The server marks a workflow over its 4 MiB read budget `limited`: no raw text, a limited graph. + await page.route('**/images/inspect/inspector/complete.png?*',async route=>{const response=await route.fetch();const data=await response.json();data.metadata.record_formats.workflow='limited';delete data.metadata.raw.workflow;data.metadata.workflow_graph={status:'limited',value:null};await route.fulfill({response,json:data});}); + await fullFixture(page,request);const panel=inspector(page); + await expect(panel.locator('.generation-facts dd')).toHaveText(['9007199254740993','28','6.5','dpmpp_2m','karras','fixture-model.safetensors']); + await expect(panel.getByRole('button',{name:'Send prompt pair',exact:true})).toBeVisible(); + await panel.getByText('Workflow & raw metadata',{exact:true}).click(); + await expect(panel.getByText('Record exceeds inspection limits; text is unavailable',{exact:true})).toBeVisible();await expect(panel.locator('pre code')).toHaveCount(1); + const workflow=panel.getByRole('button',{name:'Download workflow JSON',exact:true});await expect(workflow).toHaveAttribute('aria-disabled','true'); + await expect(workflow).toHaveAccessibleDescription('Workflow exceeds inspection limits. Download the original file to inspect it externally.'); + let downloads=0;const count=()=>downloads++;page.on('download',count);await workflow.focus();await page.keyboard.press('Enter');expect(downloads).toBe(0);page.off('download',count); + const metadata=JSON.parse((await downloadText(page,panel.getByRole('button',{name:'Download metadata JSON',exact:true}))).text); + expect(metadata.metadata.record_formats.workflow).toBe('limited');expect(metadata.metadata.raw.workflow).toBeUndefined();expect(metadata.metadata.representations.workflow).toBe('Record exceeds inspection limits; text is unavailable'); +}); + test('full inspector rejects stale shared action state',async({page,request})=>{ let reads=0,stale=false;await page.route('**/images/inspect/**',route=>{reads++;return stale?route.fulfill({status:409,json:{success:false,code:'source-changed',error:'Source changed'}}):route.continue();}); await fullFixture(page,request);const panel=inspector(page);await expect(panel.getByRole('button',{name:'Send prompt pair',exact:true})).toBeVisible();const before=reads; diff --git a/tests/fixtures/README.md b/tests/fixtures/README.md index 46040f1..25e0686 100644 --- a/tests/fixtures/README.md +++ b/tests/fixtures/README.md @@ -12,3 +12,35 @@ ffmpeg -f lavfi -i testsrc=size=32x32:rate=4:duration=1 \ ``` Browser tests use the committed file; FFmpeg is not required to run them. + +# Inspection parity fixtures + +`inspection_parity/` holds small synthetic PNGs (`f01b.png` ... `f17.png`, each +under 32 KiB) and `vectors.json`, used by the #240 inspection parity tests. Every +graph, prompt text, model name and filename prefix in them is original synthetic +content written for this repository. Nothing comes from user output. They are +dedicated to the public domain under CC0-1.0. + +`tests/inspection_parity.py` builds every fixture from code. It writes the +`prompt` and `workflow` records the way core ComfyUI `SaveImage` does (Pillow +`PngInfo.add_text`, `compress_level=4`, a 64x64 constant colour image), and +splices extra raw `tEXt` chunks with real CRC32 for the duplicate-chunk shapes +`f15` and `f16`. The larger shapes `f01`, `f04` and `f05` are built at test +time instead of committed. Reproduce the committed files with: + +```sh +python3 -m tests.inspection_parity --write tests/fixtures/inspection_parity +``` + +`tests/test_inspection_parity_fixtures.py` compares the decoded text chunks with +the builders (not bytes, because zlib output differs between platforms). + +`vectors.json` holds an `inspection` and a `default` vector for every fixture, +committed or built at test time (test-time fixtures are built into a temporary +directory). `inspection` is the workspace inspector path, with the facts the +workspace shows. `default` pins only the resolver's default profile, run on the +role records decoded from the PNG text chunks. Ingestion's own readers are covered +by their own suites. The expectations start as the observed baseline. A change +that alters an outcome edits the vector in the same commit. +`tests/js/legacy-metadata-characterization.test.js` pins what +the legacy viewers show on the same files. diff --git a/tests/fixtures/inspection_parity/f01b.png b/tests/fixtures/inspection_parity/f01b.png new file mode 100644 index 0000000..c7237f4 Binary files /dev/null and b/tests/fixtures/inspection_parity/f01b.png differ diff --git a/tests/fixtures/inspection_parity/f02.png b/tests/fixtures/inspection_parity/f02.png new file mode 100644 index 0000000..7360b8e Binary files /dev/null and b/tests/fixtures/inspection_parity/f02.png differ diff --git a/tests/fixtures/inspection_parity/f03.png b/tests/fixtures/inspection_parity/f03.png new file mode 100644 index 0000000..255c8bf Binary files /dev/null and b/tests/fixtures/inspection_parity/f03.png differ diff --git a/tests/fixtures/inspection_parity/f06a.png b/tests/fixtures/inspection_parity/f06a.png new file mode 100644 index 0000000..b724027 Binary files /dev/null and b/tests/fixtures/inspection_parity/f06a.png differ diff --git a/tests/fixtures/inspection_parity/f06b.png b/tests/fixtures/inspection_parity/f06b.png new file mode 100644 index 0000000..bb313ec Binary files /dev/null and b/tests/fixtures/inspection_parity/f06b.png differ diff --git a/tests/fixtures/inspection_parity/f06c.png b/tests/fixtures/inspection_parity/f06c.png new file mode 100644 index 0000000..da7cb3e Binary files /dev/null and b/tests/fixtures/inspection_parity/f06c.png differ diff --git a/tests/fixtures/inspection_parity/f06d.png b/tests/fixtures/inspection_parity/f06d.png new file mode 100644 index 0000000..03ce8ce Binary files /dev/null and b/tests/fixtures/inspection_parity/f06d.png differ diff --git a/tests/fixtures/inspection_parity/f06e.png b/tests/fixtures/inspection_parity/f06e.png new file mode 100644 index 0000000..e2d108d Binary files /dev/null and b/tests/fixtures/inspection_parity/f06e.png differ diff --git a/tests/fixtures/inspection_parity/f07a.png b/tests/fixtures/inspection_parity/f07a.png new file mode 100644 index 0000000..1d2a02c Binary files /dev/null and b/tests/fixtures/inspection_parity/f07a.png differ diff --git a/tests/fixtures/inspection_parity/f07b.png b/tests/fixtures/inspection_parity/f07b.png new file mode 100644 index 0000000..99a360d Binary files /dev/null and b/tests/fixtures/inspection_parity/f07b.png differ diff --git a/tests/fixtures/inspection_parity/f08.png b/tests/fixtures/inspection_parity/f08.png new file mode 100644 index 0000000..5740987 Binary files /dev/null and b/tests/fixtures/inspection_parity/f08.png differ diff --git a/tests/fixtures/inspection_parity/f09.png b/tests/fixtures/inspection_parity/f09.png new file mode 100644 index 0000000..8a6e688 Binary files /dev/null and b/tests/fixtures/inspection_parity/f09.png differ diff --git a/tests/fixtures/inspection_parity/f10.png b/tests/fixtures/inspection_parity/f10.png new file mode 100644 index 0000000..f765c9a Binary files /dev/null and b/tests/fixtures/inspection_parity/f10.png differ diff --git a/tests/fixtures/inspection_parity/f11.png b/tests/fixtures/inspection_parity/f11.png new file mode 100644 index 0000000..ad59170 Binary files /dev/null and b/tests/fixtures/inspection_parity/f11.png differ diff --git a/tests/fixtures/inspection_parity/f12.png b/tests/fixtures/inspection_parity/f12.png new file mode 100644 index 0000000..cdfec09 Binary files /dev/null and b/tests/fixtures/inspection_parity/f12.png differ diff --git a/tests/fixtures/inspection_parity/f14.png b/tests/fixtures/inspection_parity/f14.png new file mode 100644 index 0000000..61ea418 Binary files /dev/null and b/tests/fixtures/inspection_parity/f14.png differ diff --git a/tests/fixtures/inspection_parity/f15.png b/tests/fixtures/inspection_parity/f15.png new file mode 100644 index 0000000..ce8b3e1 Binary files /dev/null and b/tests/fixtures/inspection_parity/f15.png differ diff --git a/tests/fixtures/inspection_parity/f16.png b/tests/fixtures/inspection_parity/f16.png new file mode 100644 index 0000000..0c441e1 Binary files /dev/null and b/tests/fixtures/inspection_parity/f16.png differ diff --git a/tests/fixtures/inspection_parity/f17.png b/tests/fixtures/inspection_parity/f17.png new file mode 100644 index 0000000..5807ce4 Binary files /dev/null and b/tests/fixtures/inspection_parity/f17.png differ diff --git a/tests/fixtures/inspection_parity/vectors.json b/tests/fixtures/inspection_parity/vectors.json new file mode 100644 index 0000000..02ff874 --- /dev/null +++ b/tests/fixtures/inspection_parity/vectors.json @@ -0,0 +1,1300 @@ +[ + { + "name": "f01-inspection", + "fixture": "f01", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "No supported sampler pair found (saw FaceDetailer, SamplerCustomAdvanced)", + "reason_code": "no-sampler", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "ambiguous", + "value": null + }, + "steps": { + "status": "ambiguous", + "value": null + }, + "cfg": { + "status": "ambiguous", + "value": null + }, + "sampler": { + "status": "ambiguous", + "value": null + }, + "scheduler": { + "status": "ambiguous", + "value": null + }, + "model": { + "status": "ambiguous", + "value": null + } + } + } + }, + { + "name": "f01b-inspection", + "fixture": "f01b", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "No supported sampler pair found (saw FaceDetailer, SamplerCustomAdvanced)", + "reason_code": "no-sampler", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "1000" + }, + "steps": { + "status": "unresolved", + "value": null + }, + "cfg": { + "status": "not-recorded", + "value": null + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "unresolved", + "value": null + }, + "model": { + "status": "known", + "value": "flux-2-klein-9b-fp8.safetensors" + } + } + } + }, + { + "name": "f02-inspection", + "fixture": "f02", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f03-inspection", + "fixture": "f03", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f04-inspection", + "fixture": "f04", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f05-inspection", + "fixture": "f05", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f06a-inspection", + "fixture": "f06a", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": "unsupported-node", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f06b-inspection", + "fixture": "f06b", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": "unsupported-node", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f06c-inspection", + "fixture": "f06c", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": "unsupported-node", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f06d-inspection", + "fixture": "f06d", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": "unsupported-node", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f06e-inspection", + "fixture": "f06e", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": "unsupported-node", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f07a-inspection", + "fixture": "f07a", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f07b-inspection", + "fixture": "f07b", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "No supported sampler pair found (saw SamplerCustomAdvanced, CFGGuider)", + "reason_code": "no-sampler", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f08-inspection", + "fixture": "f08", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur, winter", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f09-inspection", + "fixture": "f09", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "ambiguous", + "reason": "Multiple sampler branches", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "ambiguous", + "value": null + }, + "steps": { + "status": "ambiguous", + "value": null + }, + "cfg": { + "status": "ambiguous", + "value": null + }, + "sampler": { + "status": "ambiguous", + "value": null + }, + "scheduler": { + "status": "ambiguous", + "value": null + }, + "model": { + "status": "ambiguous", + "value": null + } + } + } + }, + { + "name": "f10-inspection", + "fixture": "f10", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported latent ancestry", + "reason_code": "unsupported-node", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f11-inspection", + "fixture": "f11", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported latent ancestry", + "reason_code": "unsupported-node", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f12-inspection", + "fixture": "f12", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "absent", + "reason": "No parsed prompt metadata", + "reason_code": null, + "selection": null, + "positive": "", + "negative": "", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f14-inspection", + "fixture": "f14", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f15-inspection", + "fixture": "f15", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "25" + }, + "cfg": { + "status": "known", + "value": "6.5" + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "normal" + }, + "model": { + "status": "known", + "value": "sdxl_base.safetensors" + } + } + } + }, + { + "name": "f16-inspection", + "fixture": "f16", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "unreadable", + "state": "unresolved", + "reason": "Media metadata could not be read", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "unresolved", + "value": null + }, + "steps": { + "status": "unresolved", + "value": null + }, + "cfg": { + "status": "unresolved", + "value": null + }, + "sampler": { + "status": "unresolved", + "value": null + }, + "scheduler": { + "status": "unresolved", + "value": null + }, + "model": { + "status": "unresolved", + "value": null + } + } + } + }, + { + "name": "f17-inspection", + "fixture": "f17", + "profile": "inspection", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "No supported sampler pair found (saw SamplerCustomAdvanced)", + "reason_code": "no-sampler", + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": { + "seed": { + "status": "known", + "value": "42" + }, + "steps": { + "status": "known", + "value": "20" + }, + "cfg": { + "status": "not-recorded", + "value": null + }, + "sampler": { + "status": "known", + "value": "euler" + }, + "scheduler": { + "status": "known", + "value": "simple" + }, + "model": { + "status": "known", + "value": "flux1-dev.safetensors" + } + } + } + }, + { + "name": "f01-default", + "fixture": "f01", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Resolution limit reached", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f01b-default", + "fixture": "f01b", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "No supported sampler pair found", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f02-default", + "fixture": "f02", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f03-default", + "fixture": "f03", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Resolution limit reached", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f04-default", + "fixture": "f04", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f05-default", + "fixture": "f05", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f06a-default", + "fixture": "f06a", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f06b-default", + "fixture": "f06b", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f06c-default", + "fixture": "f06c", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f06d-default", + "fixture": "f06d", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f06e-default", + "fixture": "f06e", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported node semantics or output slot", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f07a-default", + "fixture": "f07a", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f07b-default", + "fixture": "f07b", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "No supported sampler pair found", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f08-default", + "fixture": "f08", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur, winter", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f09-default", + "fixture": "f09", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "ambiguous", + "reason": "Multiple sampler branches", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 2, + "facts": null + } + }, + { + "name": "f10-default", + "fixture": "f10", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported latent ancestry", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f11-default", + "fixture": "f11", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "unresolved", + "reason": "Unsupported latent ancestry", + "reason_code": null, + "selection": null, + "positive": null, + "negative": null, + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f12-default", + "fixture": "f12", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "absent", + "reason": "No parsed prompt metadata", + "reason_code": null, + "selection": null, + "positive": "", + "negative": "", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f14-default", + "fixture": "f14", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f15-default", + "fixture": "f15", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a red fox in snow, detailed fur", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f16-default", + "fixture": "f16", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "resolved", + "reason": "", + "reason_code": null, + "selection": null, + "positive": "a grey wolf in fog", + "negative": "blurry, lowres", + "candidate_count": 0, + "facts": null + } + }, + { + "name": "f17-default", + "fixture": "f17", + "profile": "default", + "anchor": null, + "expect": { + "metadata_status": "available", + "state": "absent", + "reason": "No supported sampler pair found", + "reason_code": null, + "selection": null, + "positive": "", + "negative": "", + "candidate_count": 0, + "facts": null + } + } +] diff --git a/tests/inspection_parity.py b/tests/inspection_parity.py new file mode 100644 index 0000000..dc33cfc --- /dev/null +++ b/tests/inspection_parity.py @@ -0,0 +1,920 @@ +"""Deterministic synthetic ComfyUI-shaped fixtures for inspection parity (#240). + +Every graph here is original synthetic content (CC0): neutral texts, generic model +file names and invented filename prefixes. Nothing is copied from user outputs. +The shapes reproduce the parity audit's fixture numbering (``f01``..``f17``). + +``build(name)`` returns ``{"prompt", "workflow", "extra_chunks"}``. ``write_png`` +writes the two records the way core ``SaveImage`` does (Pillow ``PngInfo.add_text``, +``compress_level=4``, a 64x64 constant colour image), then splices any extra raw +``tEXt`` chunks, with real CRC32, before ``IEND``. + +Regenerate the committed files with:: + + python3 -m tests.inspection_parity --write tests/fixtures/inspection_parity +""" + +import argparse +import io +import json +import struct +import tempfile +import zlib +from pathlib import Path + +from PIL import Image +from PIL.PngImagePlugin import PngInfo + +POS = "a red fox in snow, detailed fur" +NEG = "blurry, lowres" +FIXTURE_DIR = Path(__file__).resolve().parent / "fixtures" / "inspection_parity" +VECTORS_PATH = FIXTURE_DIR / "vectors.json" +MAX_FIXTURE_BYTES = 32 * 1024 +_SIGNATURE = b"\x89PNG\r\n\x1a\n" +ABSENT = "" +FACT_FIELDS = ("seed", "steps", "cfg", "sampler", "scheduler", "model") +EXPECT_KEYS = ( + "metadata_status", + "state", + "reason", + "reason_code", + "selection", + "positive", + "negative", + "candidate_count", + "facts", +) + +# Small enough to commit (each file stays under MAX_FIXTURE_BYTES). +COMMITTED = ( + "f01b", + "f02", + "f03", + "f06a", + "f06b", + "f06c", + "f06d", + "f06e", + "f07a", + "f07b", + "f08", + "f09", + "f10", + "f11", + "f12", + "f14", + "f15", + "f16", + "f17", +) +# Built in a temporary directory at test time (too large to commit). +TEST_TIME = ("f01", "f04", "f05") + + +def _ui(prompt, extra_nodes=0, subgraphs=0, big_widget=None): + """A UI workflow: one node per prompt node plus inert reroutes/notes/primitives.""" + nodes = [ + {"id": index + 1, "type": node["class_type"], "mode": 0} + for index, node in enumerate(prompt.values()) + ] + base = len(nodes) + for offset in range(extra_nodes): + kind = ("Reroute", "Note", "PrimitiveNode")[offset % 3] + nodes.append({"id": base + offset + 1, "type": kind, "mode": 0}) + if big_widget is not None: + nodes.append( + {"id": 999999, "type": "Note", "mode": 0, "widgets_values": [big_widget]} + ) + workflow = {"last_node_id": len(nodes), "nodes": nodes, "links": [], "version": 0.4} + if subgraphs: + workflow["definitions"] = { + "subgraphs": [ + {"id": f"sg-{index}", "nodes": [], "links": []} + for index in range(subgraphs) + ] + } + return workflow + + +def _enc(text): + return {"class_type": "CLIPTextEncode", "inputs": {"text": text, "clip": ["4", 1]}} + + +def _ksampler_graph(positive, negative, sampler="KSampler", latent=None, extra=None): + graph = { + "4": { + "class_type": "CheckpointLoaderSimple", + "inputs": {"ckpt_name": "sdxl_base.safetensors"}, + }, + "5": latent + or { + "class_type": "EmptyLatentImage", + "inputs": {"width": 1024, "height": 1024, "batch_size": 1}, + }, + "6": positive, + "7": negative, + "3": { + "class_type": sampler, + "inputs": { + "seed": 42, + "steps": 25, + "cfg": 6.5, + "sampler_name": "euler", + "scheduler": "normal", + "denoise": 1.0, + "model": ["4", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "latent_image": ["5", 0], + }, + }, + "8": { + "class_type": "VAEDecode", + "inputs": {"samples": ["3", 0], "vae": ["4", 2]}, + }, + "9": { + "class_type": "SaveImage", + "inputs": {"filename_prefix": "ComfyUI", "images": ["8", 0]}, + }, + } + if sampler == "KSamplerAdvanced": + inputs = graph["3"]["inputs"] + inputs["noise_seed"] = inputs.pop("seed") + inputs.pop("denoise") + inputs.update( + add_noise="enable", + start_at_step=0, + end_at_step=10000, + return_with_leftover_noise="disable", + ) + graph.update(extra or {}) + return graph + + +def _basic(prompt, workflow=None, extra_chunks=()): + return { + "prompt": prompt, + "workflow": workflow, + "extra_chunks": [tuple(chunk) for chunk in extra_chunks], + } + + +def _flux2_branches(branches, gates=4): + """Subgraph-expanded flux2 shape: N SaveImage branches plus a preview-only detailer. + + Each branch is SamplerCustomAdvanced <- BasicGuider <- (easy ifElse / + ReferenceLatent) x gates <- FluxGuidance <- CLIPTextEncode <- JoinStringMulti. + Every SaveImage shares one JoinStringMulti ``filename_prefix``. A FaceDetailer + with a different prompt feeds only a PreviewImage. + """ + graph = { + "11": { + "class_type": "VAELoader", + "inputs": {"vae_name": "vae.safetensors"}, + }, + "12": { + "class_type": "CLIPLoader", + "inputs": {"clip_name": "text_encoder.safetensors", "type": "flux2"}, + }, + "13": { + "class_type": "UNETLoader", + "inputs": { + "unet_name": "flux-2-klein-9b-fp8.safetensors", + "weight_dtype": "default", + }, + }, + "14": {"class_type": "LoadImage", "inputs": {"image": "reference.png"}}, + "21": { + "class_type": "StringConstantMultiline", + "inputs": {"string": POS + ",", "strip_newlines": False}, + }, + "22": { + "class_type": "StringConstantMultiline", + "inputs": {"string": "soft winter light,", "strip_newlines": False}, + }, + "31": {"class_type": "PrimitiveString", "inputs": {"value": "Project"}}, + "32": {"class_type": "PrimitiveString", "inputs": {"value": "fixture"}}, + "33": {"class_type": "PrimitiveString", "inputs": {"value": "IMG"}}, + "34": { + "class_type": "JoinStringMulti", + "inputs": { + "inputcount": 2, + "delimiter": "_", + "return_list": False, + "string_1": ["32", 0], + "string_2": ["33", 0], + }, + }, + "35": { + "class_type": "JoinStringMulti", + "inputs": { + "inputcount": 4, + "delimiter": "/", + "return_list": False, + "string_1": ["31", 0], + "string_2": ["32", 0], + "string_3": ["34", 0], + "string_4": ["34", 0], + }, + }, + "41": { + "class_type": "PrimitiveStringMultiline", + "inputs": {"value": "Detailer pass: refine the face"}, + }, + "42": { + "class_type": "CLIPTextEncode", + "inputs": {"text": ["41", 0], "clip": ["12", 0]}, + }, + "43": { + "class_type": "ConditioningZeroOut", + "inputs": {"conditioning": ["42", 0]}, + }, + "44": { + "class_type": "FaceDetailer", + "inputs": { + "image": ["14", 0], + "model": ["13", 0], + "clip": ["12", 0], + "vae": ["11", 0], + "positive": ["42", 0], + "negative": ["43", 0], + "seed": 271828182845, + "steps": 20, + "cfg": 1, + "sampler_name": "euler", + "scheduler": "simple", + }, + }, + "45": {"class_type": "PreviewImage", "inputs": {"images": ["44", 0]}}, + } + for branch in range(branches): + scope = f"{1300 + branch}" + + def sub(local, scope=scope): + return f"{scope}:{local}" + + graph[f"{1100 + branch}"] = { + "class_type": "StringConstantMultiline", + "inputs": { + "string": f"Scene {branch}: unique shot description", + "strip_newlines": False, + }, + } + graph[f"{1200 + branch}"] = { + "class_type": "JoinStringMulti", + "inputs": { + "inputcount": 3, + "delimiter": " ", + "return_list": False, + "string_1": ["21", 0], + "string_2": [f"{1100 + branch}", 0], + "string_3": ["22", 0], + }, + } + graph[sub(1)] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": [f"{1200 + branch}", 0], "clip": ["12", 0]}, + } + graph[sub(2)] = { + "class_type": "FluxGuidance", + "inputs": {"guidance": 3.5, "conditioning": [sub(1), 0]}, + } + previous = sub(2) + for gate in range(gates): + graph[sub(10 + gate)] = { + "class_type": "ImageExists", + "inputs": {"image": ["14", 0]}, + } + graph[sub(20 + gate)] = { + "class_type": "VAEEncode", + "inputs": {"pixels": [sub(10 + gate), 0], "vae": ["11", 0]}, + } + graph[sub(30 + gate)] = { + "class_type": "ReferenceLatent", + "inputs": { + "conditioning": [previous, 0], + "latent": [sub(20 + gate), 0], + }, + } + graph[sub(40 + gate)] = { + "class_type": "easy ifElse", + "inputs": { + "boolean": [sub(10 + gate), 1], + "on_true": [sub(30 + gate), 0], + "on_false": [previous, 0], + }, + } + previous = sub(40 + gate) + graph[sub(3)] = { + "class_type": "BasicGuider", + "inputs": {"model": ["13", 0], "conditioning": [previous, 0]}, + } + graph[sub(4)] = { + "class_type": "RandomNoise", + "inputs": {"noise_seed": 1000 + branch}, + } + graph[sub(5)] = { + "class_type": "KSamplerSelect", + "inputs": {"sampler_name": "euler"}, + } + graph[sub(6)] = { + "class_type": "Flux2Scheduler", + "inputs": {"steps": 20, "width": 1024, "height": 1024}, + } + graph[sub(7)] = { + "class_type": "EmptyFlux2LatentImage", + "inputs": {"width": 1024, "height": 1024, "batch_size": 1}, + } + graph[sub(8)] = { + "class_type": "SamplerCustomAdvanced", + "inputs": { + "noise": [sub(4), 0], + "guider": [sub(3), 0], + "sampler": [sub(5), 0], + "sigmas": [sub(6), 0], + "latent_image": [sub(7), 0], + }, + } + graph[sub(9)] = { + "class_type": "VAEDecode", + "inputs": {"samples": [sub(8), 0], "vae": ["11", 0]}, + } + graph[f"{1400 + branch}"] = { + "class_type": "SaveImage", + "inputs": {"filename_prefix": ["35", 0], "images": [sub(9), 0]}, + } + return graph + + +def _top_level(graph): + return {key: node for key, node in graph.items() if ":" not in key} + + +def _f01(): + graph = _flux2_branches(24) + return _basic(graph, _ui(_top_level(graph), 450, subgraphs=24)) + + +def _f01b(): + graph = _flux2_branches(1) + return _basic(graph, _ui(_top_level(graph), subgraphs=1)) + + +def _f02(): + graph = _ksampler_graph(_enc(POS), _enc(NEG)) + return _basic(graph, _ui(graph, 600)) + + +def _f03(): + side = { + f"{100 + index}": {"class_type": "PrimitiveInt", "inputs": {"value": 1}} + for index in range(499) + } + return _basic(_ksampler_graph(_enc(POS), _enc(NEG), extra=side)) + + +def _f04(): + graph = _ksampler_graph(_enc(POS), _enc(NEG)) + return _basic(graph, _ui(graph, big_widget="A" * 2_500_000)) + + +def _f05(): + graph = _ksampler_graph(_enc(POS), _enc(NEG)) + return _basic(graph, _ui(graph, big_widget='"' * 900_000)) + + +def _f06a(): + sdxl = { + "class_type": "CLIPTextEncodeSDXL", + "inputs": { + "text_g": POS, + "text_l": POS, + "width": 1024, + "height": 1024, + "crop_w": 0, + "crop_h": 0, + "target_width": 1024, + "target_height": 1024, + "clip": ["4", 1], + }, + } + return _basic(_ksampler_graph(sdxl, _enc(NEG))) + + +def _f06b(): + flux = { + "class_type": "CLIPTextEncodeFlux", + "inputs": {"clip_l": POS, "t5xxl": POS, "guidance": 3.5, "clip": ["4", 1]}, + } + return _basic(_ksampler_graph(flux, _enc(NEG))) + + +def _f06c(): + guidance = { + "class_type": "FluxGuidance", + "inputs": {"guidance": 3.5, "conditioning": ["10", 0]}, + } + return _basic(_ksampler_graph(guidance, _enc(NEG), extra={"10": _enc(POS)})) + + +def _f06d(): + combine = { + "class_type": "ConditioningCombine", + "inputs": {"conditioning_1": ["10", 0], "conditioning_2": ["11", 0]}, + } + extra = {"10": _enc(POS), "11": _enc("a second subject")} + return _basic(_ksampler_graph(combine, _enc(NEG), extra=extra)) + + +def _f06e(): + def apply(): + return { + "class_type": "ControlNetApplyAdvanced", + "inputs": { + "positive": ["10", 0], + "negative": ["11", 0], + "control_net": ["12", 0], + "image": ["13", 0], + "strength": 1.0, + "start_percent": 0, + "end_percent": 1, + }, + } + + extra = { + "10": _enc(POS), + "11": _enc(NEG), + "12": { + "class_type": "ControlNetLoader", + "inputs": {"control_net_name": "control.safetensors"}, + }, + "13": {"class_type": "LoadImage", "inputs": {"image": "control.png"}}, + } + graph = _ksampler_graph(apply(), apply(), extra=extra) + graph["3"]["inputs"]["negative"] = ["7", 1] + return _basic(graph) + + +def _f07a(): + return _basic(_ksampler_graph(_enc(POS), _enc(NEG), "KSamplerAdvanced")) + + +def _f07b(): + graph = _ksampler_graph(_enc(POS), _enc(NEG)) + graph["3"] = { + "class_type": "SamplerCustomAdvanced", + "inputs": { + "noise": ["20", 0], + "guider": ["21", 0], + "sampler": ["22", 0], + "sigmas": ["23", 0], + "latent_image": ["5", 0], + }, + } + graph.update( + { + "20": {"class_type": "RandomNoise", "inputs": {"noise_seed": 42}}, + "21": { + "class_type": "CFGGuider", + "inputs": { + "model": ["4", 0], + "positive": ["6", 0], + "negative": ["7", 0], + "cfg": 6.5, + }, + }, + "22": {"class_type": "KSamplerSelect", "inputs": {"sampler_name": "euler"}}, + "23": { + "class_type": "BasicScheduler", + "inputs": { + "model": ["4", 0], + "scheduler": "normal", + "steps": 25, + "denoise": 1.0, + }, + }, + } + ) + return _basic(graph) + + +def _f08(): + extra = { + "30": { + "class_type": "StringConcatenate", + "inputs": {"string_a": ["31", 0], "string_b": ", winter", "delimiter": ""}, + }, + "31": {"class_type": "PrimitiveStringMultiline", "inputs": {"value": POS}}, + } + return _basic(_ksampler_graph(_enc(["30", 0]), _enc(NEG), extra=extra)) + + +def _f09(): + graph = _ksampler_graph(_enc(POS), _enc(NEG)) + graph["40"] = { + "class_type": "LatentUpscaleBy", + "inputs": { + "upscale_method": "nearest-exact", + "scale_by": 1.5, + "samples": ["3", 0], + }, + } + graph["41"] = { + "class_type": "KSampler", + "inputs": {**graph["3"]["inputs"], "latent_image": ["40", 0], "denoise": 0.5}, + } + graph["8"]["inputs"]["samples"] = ["41", 0] + return _basic(graph) + + +def _f10(): + latent = { + "class_type": "VAEEncode", + "inputs": {"pixels": ["50", 0], "vae": ["4", 2]}, + } + extra = {"50": {"class_type": "LoadImage", "inputs": {"image": "input.png"}}} + return _basic(_ksampler_graph(_enc(POS), _enc(NEG), latent=latent, extra=extra)) + + +def _f11(): + latent = { + "class_type": "EmptyHunyuanLatentVideo", + "inputs": {"width": 640, "height": 384, "length": 17, "batch_size": 1}, + } + return _basic(_ksampler_graph(_enc(POS), _enc(NEG), latent=latent)) + + +def _f12(): + def link(name, link_id): + return {"name": name, "link": link_id} + + workflow = { + "nodes": [ + { + "id": 3, + "type": "KSampler", + "mode": 0, + "inputs": [ + link("model", 1), + link("positive", 2), + link("negative", 3), + link("latent_image", 4), + ], + "widgets_values": [42, "fixed", 25, 6.5, "euler", "normal", 1.0], + }, + { + "id": 4, + "type": "CheckpointLoaderSimple", + "mode": 0, + "inputs": [], + "widgets_values": ["sdxl_base.safetensors"], + }, + { + "id": 6, + "type": "CLIPTextEncode", + "mode": 0, + "inputs": [], + "widgets_values": [POS], + }, + { + "id": 7, + "type": "CLIPTextEncode", + "mode": 0, + "inputs": [], + "widgets_values": [NEG], + }, + { + "id": 5, + "type": "EmptyLatentImage", + "mode": 0, + "inputs": [], + "widgets_values": [1024, 1024, 1], + }, + ], + "links": [ + [1, 4, 0, 3, 0, "MODEL"], + [2, 6, 0, 3, 1, "CONDITIONING"], + [3, 7, 0, 3, 2, "CONDITIONING"], + [4, 5, 0, 3, 3, "LATENT"], + ], + } + return _basic(None, workflow) + + +def _f14(): + """ComfyUI writes ``_meta`` titles and ``"is_changed": [NaN]`` outside inputs.""" + graph = _ksampler_graph(_enc(POS), _enc(NEG)) + for node in graph.values(): + node["_meta"] = {"title": node["class_type"]} + graph["4"]["is_changed"] = [float("nan")] + return _basic(graph, _ui(graph)) + + +def _f15(): + """Byte-identical duplicate ``prompt``/``workflow`` chunks (four copies each).""" + graph = _ksampler_graph(_enc(POS), _enc(NEG)) + workflow = _ui(graph) + prompt_text, workflow_text = json.dumps(graph), json.dumps(workflow) + extra = [("prompt", prompt_text), ("workflow", workflow_text)] * 3 + return _basic(graph, workflow, extra) + + +def _f16(): + """A second ``prompt`` chunk that differs from the first.""" + graph = _ksampler_graph(_enc(POS), _enc(NEG)) + other = _ksampler_graph(_enc("a grey wolf in fog"), _enc(NEG)) + return _basic(graph, _ui(graph), [("prompt", json.dumps(other))]) + + +def _f17(): + """SamplerCustomAdvanced + BasicGuider only: no positive/negative inputs.""" + graph = { + "10": { + "class_type": "UNETLoader", + "inputs": {"unet_name": "flux1-dev.safetensors", "weight_dtype": "default"}, + }, + "11": { + "class_type": "DualCLIPLoader", + "inputs": { + "clip_name1": "clip_l.safetensors", + "clip_name2": "t5xxl.safetensors", + "type": "flux", + }, + }, + "12": {"class_type": "VAELoader", "inputs": {"vae_name": "ae.safetensors"}}, + "6": { + "class_type": "CLIPTextEncode", + "inputs": {"text": POS, "clip": ["11", 0]}, + }, + "26": { + "class_type": "FluxGuidance", + "inputs": {"guidance": 3.5, "conditioning": ["6", 0]}, + }, + "22": { + "class_type": "BasicGuider", + "inputs": {"model": ["10", 0], "conditioning": ["26", 0]}, + }, + "25": {"class_type": "RandomNoise", "inputs": {"noise_seed": 42}}, + "16": {"class_type": "KSamplerSelect", "inputs": {"sampler_name": "euler"}}, + "17": { + "class_type": "BasicScheduler", + "inputs": { + "model": ["10", 0], + "scheduler": "simple", + "steps": 20, + "denoise": 1.0, + }, + }, + "27": { + "class_type": "EmptySD3LatentImage", + "inputs": {"width": 1024, "height": 1024, "batch_size": 1}, + }, + "13": { + "class_type": "SamplerCustomAdvanced", + "inputs": { + "noise": ["25", 0], + "guider": ["22", 0], + "sampler": ["16", 0], + "sigmas": ["17", 0], + "latent_image": ["27", 0], + }, + }, + "8": { + "class_type": "VAEDecode", + "inputs": {"samples": ["13", 0], "vae": ["12", 0]}, + }, + "9": { + "class_type": "SaveImage", + "inputs": {"filename_prefix": "ComfyUI", "images": ["8", 0]}, + }, + } + return _basic(graph, _ui(graph)) + + +_BUILDERS = { + "f01": _f01, + "f01b": _f01b, + "f02": _f02, + "f03": _f03, + "f04": _f04, + "f05": _f05, + "f06a": _f06a, + "f06b": _f06b, + "f06c": _f06c, + "f06d": _f06d, + "f06e": _f06e, + "f07a": _f07a, + "f07b": _f07b, + "f08": _f08, + "f09": _f09, + "f10": _f10, + "f11": _f11, + "f12": _f12, + "f14": _f14, + "f15": _f15, + "f16": _f16, + "f17": _f17, +} + + +def build(name): + """Return a fresh ``{"prompt", "workflow", "extra_chunks"}`` for one fixture.""" + try: + builder = _BUILDERS[name] + except KeyError: + raise KeyError(f"Unknown inspection parity fixture: {name}") from None + return builder() + + +def fixture_path(name, directory=FIXTURE_DIR): + return Path(directory) / f"{name}.png" + + +def expected_text_chunks(built): + """The decoded ``(keyword, text)`` chunks ``write_png`` emits, in file order.""" + chunks = [] + if built["prompt"] is not None: + chunks.append(("prompt", json.dumps(built["prompt"]))) + if built["workflow"] is not None: + chunks.append(("workflow", json.dumps(built["workflow"]))) + return chunks + [tuple(chunk) for chunk in built["extra_chunks"]] + + +def _raw_chunk(kind, data): + return ( + struct.pack(">I", len(data)) + + kind + + data + + struct.pack(">I", zlib.crc32(kind + data) & 0xFFFFFFFF) + ) + + +def write_png(path, built): + """Write ``built`` as core SaveImage does, then splice extra raw text chunks.""" + info = PngInfo() + if built["prompt"] is not None: + info.add_text("prompt", json.dumps(built["prompt"])) + if built["workflow"] is not None: + info.add_text("workflow", json.dumps(built["workflow"])) + buffer = io.BytesIO() + with Image.new("RGB", (64, 64), (40, 80, 120)) as image: + image.save(buffer, format="PNG", pnginfo=info, compress_level=4) + data = buffer.getvalue() + iend = data.rindex(b"IEND") - 4 + extra = b"".join( + _raw_chunk(b"tEXt", key.encode("latin-1") + b"\0" + text.encode("latin-1")) + for key, text in built["extra_chunks"] + ) + with open(path, "wb") as stream: + stream.write(data[:iend] + extra + data[iend:]) + + +def write_all(directory, names=COMMITTED): + directory = Path(directory) + directory.mkdir(parents=True, exist_ok=True) + for name in names: + write_png(fixture_path(name, directory), build(name)) + return [fixture_path(name, directory) for name in names] + + +def read_text_chunks(path): + """Decode every tEXt/zTXt/iTXt chunk as ``(keyword, text)``, duplicates kept.""" + with open(path, "rb") as stream: + data = stream.read() + if not data.startswith(_SIGNATURE): + raise ValueError(f"Not a PNG: {path}") + chunks, offset = [], len(_SIGNATURE) + while offset < len(data): + length, kind = struct.unpack(">I4s", data[offset : offset + 8]) + body = data[offset + 8 : offset + 8 + length] + offset += 12 + length + if kind == b"tEXt": + key, _, text = body.partition(b"\0") + chunks.append((key.decode("latin-1"), text.decode("latin-1"))) + elif kind == b"zTXt": + key, _, rest = body.partition(b"\0") + chunks.append( + (key.decode("latin-1"), zlib.decompress(rest[1:]).decode("latin-1")) + ) + elif kind == b"iTXt": + key, _, rest = body.partition(b"\0") + flag, fields = rest[0], rest[2:].split(b"\0", 2) + text = zlib.decompress(fields[2]) if flag else fields[2] + chunks.append((key.decode("latin-1"), text.decode("utf-8"))) + elif kind == b"IEND": + break + return chunks + + +def load_vectors(path=VECTORS_PATH): + with open(path, encoding="utf-8") as stream: + return json.load(stream) + + +def _role_text(status, text): + if status == "resolved": + return text + return ABSENT if status == "absent" else None + + +def _observe_inspection(read): + from utils.prompt_inspector import inspect_prompt_evidence + from utils.prompt_pair_resolver import INSPECTION_PROFILE + + if read is None: # inspect_current_file's unreadable-media evidence + return "unresolved", "Media metadata could not be read", None, None, {} + evidence = inspect_prompt_evidence(read.metadata, profile=INSPECTION_PROFILE) + resolution, prompts = evidence["resolution"], evidence["prompts"] + roles = [ + _role_text(prompts[role]["status"], prompts[role]["text"]) + for role in ("positive", "negative") + ] + positive, negative = roles + return resolution["state"], resolution["reason"], positive, negative, resolution + + +def _observe_default(path): + from utils.prompt_pair_resolver import DEFAULT_PROFILE, resolve_prompt_pair + + # Last chunk wins, as Pillow's ``image.text`` (the metadata extractor) does. + records = dict(read_text_chunks(path)) + canonical = { + key: value for key, value in records.items() if key in ("prompt", "parameters") + } + if isinstance(canonical.get("prompt"), str): + canonical["prompt"] = json.loads(canonical["prompt"]) + resolved = resolve_prompt_pair(canonical, profile=DEFAULT_PROFILE) + state = resolved["state"] + roles = [ + ( + resolved[role]["text"] + if state == "resolved" and isinstance(resolved[role], dict) + else (ABSENT if state == "absent" else None) + ) + for role in ("positive", "negative") + ] + resolved = {**resolved, "candidate_count": len(resolved.get("candidates") or [])} + positive, negative = roles + return "available", state, resolved.get("reason", ""), positive, negative, resolved + + +def _observe_path(vector, path): + from utils.prompt_inspector import MetadataReadError, read_inspection_file + + if vector["profile"] != "inspection": + return _observe_default(path) + try: + read, status = read_inspection_file(path), "available" + except MetadataReadError: + read, status = None, "unreadable" + state, reason, positive, negative, resolution = _observe_inspection(read) + return status, state, reason, positive, negative, resolution + + +def observe(vector, directory=FIXTURE_DIR): + """Today's outcome for one vector, shaped like ``vector["expect"]`` minus facts. + + ``inspection`` is the workspace inspector path (``read_inspection_file`` then + ``inspect_prompt_evidence``) with ``INSPECTION_PROFILE``. ``default`` pins + the resolver's ``DEFAULT_PROFILE`` only: ``resolve_prompt_pair`` on the role + records decoded from the PNG text chunks. Ingestion's own readers are + covered by their own suites. + + Committed fixtures are read from ``directory``, which must contain them. + Test-time fixtures are built into a temporary directory; the file is + written and closed before it is read, and closed again before cleanup. + """ + name = vector["fixture"] + if name in TEST_TIME: + with tempfile.TemporaryDirectory() as temporary: + path = fixture_path(name, temporary) + write_png(path, build(name)) + observed = _observe_path(vector, path) + else: + path = fixture_path(name, directory) + if not path.is_file(): + raise FileNotFoundError(path) + observed = _observe_path(vector, path) + status, state, reason, positive, negative, resolution = observed + return { + "metadata_status": status, + "state": state, + "reason": reason, + "reason_code": resolution.get("reason_code"), + "selection": resolution.get("selection"), + "positive": positive, + "negative": negative, + "candidate_count": resolution.get("candidate_count", 0), + } + + +def main(argv=None): + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--write", required=True, help="output directory") + args = parser.parse_args(argv) + for written in write_all(args.write): + print(written) + + +if __name__ == "__main__": + main() diff --git a/tests/js/comfy-metadata.test.js b/tests/js/comfy-metadata.test.js index 451d6b2..7d64f08 100644 --- a/tests/js/comfy-metadata.test.js +++ b/tests/js/comfy-metadata.test.js @@ -380,11 +380,33 @@ test("conservative facts bound cycles fan-out visits and link depth", () => { custom.s = { class_type: "SamplerCustomAdvanced", inputs: { noise: ["noise", 0] } }; custom.noise = { class_type: "RandomNoise", inputs: { noise_seed: ["p0", 0] } }; assert.equal(factsFor(custom).seed.status, "unresolved", "the provider itself consumes one of the 64 allowed links"); +}); + +// Spec section 10: graph size is not a facts bound. Only the per-lookup visit +// set (MAX_VISITS) and the projection's 20,000-node ceiling (`limited`) bound work. +test("facts resolve on a parsed 600-node graph with one KSampler", () => { const large = factGraph(); - for (let index = 0; index < 498; index++) large[`unused${index}`] = { class_type: "PrimitiveInt", inputs: { value: index } }; - assert.equal(factsFor(large).seed.value, "9007199254740993"); - large.extra = { class_type: "PrimitiveInt", inputs: { value: 1 } }; - assert.equal(factsFor(large).seed.status, "unresolved"); + for (let index = 0; index < 598; index++) large[`unused${index}`] = { class_type: "PrimitiveInt", inputs: { value: index } }; + assert.equal(Object.keys(large).length, 600); + const facts = factsFor(large); + for (const [field, value] of [["seed", "9007199254740993"], ["steps", "28"], ["cfg", "6.5000000000000000001"]]) { + assert.equal(facts[field].status, "known", field); + assert.equal(facts[field].value, value); + assert.equal(facts[field].reason, null); + } +}); + +test("workflow facts are not gated by UI node count", () => { + const nodes = [ + { id: 1, type: "CheckpointLoaderSimple", widgets_values: ["wide.safetensors"] }, + { id: 2, type: "KSampler", widgets_values: [42, "fixed", 20, "7", "euler", "normal"], inputs: [{ name: "model", link: 1 }] }, + ]; + for (let index = 0; index < 598; index++) nodes.push({ id: `note${index}`, type: "Note", widgets_values: [`note ${index}`] }); + const workflow = { nodes, links: [[1, 1, 0, 2, 0, "MODEL"]] }; + const facts = factsFor(null, { prompt_graph: absentFactGraph(), workflow_graph: parsedFactGraph(workflow) }); + assert.deepEqual(factFields.map(field => facts[field].status), Array(6).fill("known")); + assert.deepEqual(factFields.map(field => facts[field].value), ["42", "20", "7", "euler", "normal", "wide.safetensors"]); + assert.ok(Object.values(facts).every(fact => fact.source === "workflow" && fact.reason === null)); }); // Same shape as the workflow attached to #75: PromptManager nodes are saved as diff --git a/tests/js/legacy-metadata-characterization.test.js b/tests/js/legacy-metadata-characterization.test.js new file mode 100644 index 0000000..cb1ed37 --- /dev/null +++ b/tests/js/legacy-metadata-characterization.test.js @@ -0,0 +1,82 @@ +// Run with: node --test tests/js/ +// Characterization, not a specification: pins what the legacy viewers +// (web/metadata.html, web/js/gallery.js) show today on the synthetic inspection +// parity fixtures, so the #240 workspace work cannot change them by accident. +// Some answers are known to be wrong (f01b shows the preview-only detailer text +// as both roles; f16 shows the last of two conflicting prompt chunks). They are +// recorded as-is and intentionally unchanged (spec section 11). +const test = require("node:test"); +const assert = require("node:assert/strict"); +const fs = require("node:fs"); +const path = require("node:path"); +const { parsePngTextChunks } = require("../../web/js/png-metadata.js"); +const { extractGenerationParams } = require("../../web/js/comfy-metadata.js"); + +const FIXTURES = path.join(__dirname, "..", "fixtures", "inspection_parity"); +const DETAILER = "Detailer pass: refine the face"; + +// Copy of the legacy pages' inline glue (NaN cleanup and field order): web/metadata.html +// extractComfyUIData (line 107) and web/js/gallery.js extractComfyUIData (line 1643). +// The spec only promises extractGenerationParams is unchanged, so the glue is copied here. +function legacyComfyData(metadata) { + const parse = text => JSON.parse(text.replace(/:\s*NaN\b/g, ": null").replace(/\bNaN\b/g, "null")); + const first = fields => { + for (const field of fields) { + if (!metadata[field]) continue; + try { return parse(metadata[field]); } catch { /* the pages try the next field */ } + } + return null; + }; + return { workflow: first(["workflow", "Workflow", "comfy", "ComfyUI"]), prompt: first(["prompt", "Prompt", "parameters", "Parameters"]) }; +} + +function legacyParams(name) { + const bytes = fs.readFileSync(path.join(FIXTURES, `${name}.png`)); + const buffer = bytes.buffer.slice(bytes.byteOffset, bytes.byteOffset + bytes.byteLength); + return extractGenerationParams(legacyComfyData(parsePngTextChunks(buffer) || {})); +} + +const EXPECTED = { + f01b: { positivePrompt: "Detailer pass: refine the face", negativePrompt: "Detailer pass: refine the face", checkpoint: "flux-2-klein-9b-fp8.safetensors", steps: 20, cfgScale: 1, sampler: "euler", seed: 271828182845 }, + f02: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f03: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f06a: { positivePrompt: "No prompt found", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f06b: { positivePrompt: "No prompt found", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f06c: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f06d: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f06e: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f07a: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f07b: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f08: { positivePrompt: "a red fox in snow, detailed fur, winter", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f09: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f10: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f11: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f12: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f14: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f15: { positivePrompt: "a red fox in snow, detailed fur", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f16: { positivePrompt: "a grey wolf in fog", negativePrompt: "blurry, lowres", checkpoint: "sdxl_base.safetensors", steps: 25, cfgScale: 6.5, sampler: "euler", seed: 42 }, + f17: { positivePrompt: "No prompt found", negativePrompt: "No negative prompt found", checkpoint: "flux1-dev.safetensors", steps: "Unknown", cfgScale: "Unknown", sampler: "euler", seed: "Unknown" }, +}; + +test("legacy extractGenerationParams characterization covers every committed fixture", () => { + const committed = fs.readdirSync(FIXTURES).filter(file => file.endsWith(".png")).map(file => file.slice(0, -4)).sort(); + assert.deepEqual(Object.keys(EXPECTED).sort(), committed); +}); + +for (const [name, expected] of Object.entries(EXPECTED)) { + test(`legacy extractGenerationParams characterization: ${name}`, () => { + assert.deepEqual(legacyParams(name), expected); + }); +} + +test("legacy extractGenerationParams characterization keeps f01b's detailer text as both roles", () => { + const params = legacyParams("f01b"); + assert.equal(params.positivePrompt, DETAILER); + assert.equal(params.negativePrompt, DETAILER); +}); + +test("legacy extractGenerationParams characterization replaces NaN before parsing", () => { + const params = legacyParams("f14"); + assert.equal(params.positivePrompt, "a red fox in snow, detailed fur"); + assert.equal(params.seed, 42); +}); diff --git a/tests/js/support/inspection-parity-facts.mjs b/tests/js/support/inspection-parity-facts.mjs new file mode 100644 index 0000000..8a8bb07 --- /dev/null +++ b/tests/js/support/inspection-parity-facts.mjs @@ -0,0 +1,25 @@ +// Facts for encoded inspection responses, through the workspace's own path: +// projectInspectionPayload validates the wire, then extractGenerationFacts reads +// the bounded projection. Used by tests/test_inspection_parity_facts.py, which +// encodes each parity vector's file with the real route pipeline. +// stdin: [{name, body}] (body is the exact response JSON text). +// stdout: {name: {field: {status, value}}}. +import { createRequire } from 'node:module'; +import { fileURLToPath, pathToFileURL } from 'node:url'; +import path from 'node:path'; + +const here = path.dirname(fileURLToPath(import.meta.url)); +const web = path.join(here, '..', '..', '..', 'web', 'js'); +const require = createRequire(import.meta.url); +const { extractGenerationFacts } = require(path.join(web, 'comfy-metadata.js')); +const { projectInspectionPayload } = await import(pathToFileURL(path.join(web, 'workspace', 'inspection-state.mjs')).href); + +const chunks = []; +for await (const chunk of process.stdin) chunks.push(chunk); +const output = {}; +for (const { name, body } of JSON.parse(Buffer.concat(chunks).toString('utf8'))) { + const identity = { kind: 'rooted-file', key: `0:${name}.png`, root_index: 0, path: `${name}.png`, root_scope: null }; + const record = projectInspectionPayload(JSON.parse(body), identity, null, extractGenerationFacts); + output[name] = Object.fromEntries(Object.entries(record.generation).map(([field, fact]) => [field, { status: fact.status, value: fact.value }])); +} +process.stdout.write(JSON.stringify(output)); diff --git a/tests/js/workspace-file-inspector.test.js b/tests/js/workspace-file-inspector.test.js index d97be19..d129814 100644 --- a/tests/js/workspace-file-inspector.test.js +++ b/tests/js/workspace-file-inspector.test.js @@ -54,3 +54,14 @@ test('reported location metadata export carries exact occurrence without assigni source.identity={kind:'reported-location',key:'b'.repeat(64),path:'folder/a.png',occurrence:{run_id:'22222222-2222-2222-2222-222222222222',output_node_id:'𐀀',image_ordinal:255},history_epoch:'11111111-1111-1111-1111-111111111111',current_scope:'c'.repeat(64),reference:{folder_type:'temp',subfolder:'folder',filename:'a.png',path_style:'posix',root_key:null}}; const value=JSON.parse(inspectionDownloads(source).metadata.bodyText);assert.deepEqual(value.identity,source.identity);assert.equal(value.identity.root_index,undefined);assert.equal(value.context_id,undefined);assert.equal(value.file_identity,undefined);assert.deepEqual(value.prompts,source.prompts); }); + +test('limited workflow record exports its representation and disables workflow download',async()=>{ + const {inspectionDownloads}=await load(),source=record(); + source.metadata.record_formats.workflow='limited';delete source.metadata.raw.workflow;source.metadata.workflow_graph={status:'limited',value:null}; + const downloads=inspectionDownloads(source),value=JSON.parse(downloads.metadata.bodyText); + assert.equal(value.metadata.record_formats.workflow,'limited'); + assert.equal(value.metadata.representations.workflow,'Record exceeds inspection limits; text is unavailable'); + assert.equal(value.metadata.raw.workflow,undefined); + assert.equal(downloads.workflow.available,false);assert.equal(downloads.workflow.bodyText,null); + assert.equal(downloads.workflow.reason,'Workflow exceeds inspection limits. Download the original file to inspect it externally.'); +}); diff --git a/tests/js/workspace-inspection-state.test.js b/tests/js/workspace-inspection-state.test.js index f49c869..f327e23 100644 --- a/tests/js/workspace-inspection-state.test.js +++ b/tests/js/workspace-inspection-state.test.js @@ -136,7 +136,9 @@ test('record rejects contradictory omission provenance before extracting facts', data => { data.metadata.record_formats.workflow = 'original-text'; data.metadata.workflow_graph = { status: 'parsed', value: { nodes: [] } }; }, data => { Object.assign(data.metadata, { status: 'limited', omitted_reason: 'response-size-limit', raw: {}, prompt_graph: { status: 'limited', value: null } }); data.metadata.record_formats.workflow = 'invalid'; data.metadata.workflow_graph = { status: 'malformed', value: null }; }, data => { data.metadata.record_formats.workflow = 'invalid'; data.metadata.raw.workflow = 'null'; data.metadata.workflow_graph = { status: 'limited', value: null }; }, - data => { data.metadata.record_formats.workflow = 'invalid'; data.metadata.workflow_graph = { status: 'limited', value: null }; }, + // The response budget omits the workflow graph before the prompt graph, so an + // omitted invalid prompt graph means the workflow graph was omitted too. + data => { data.metadata.record_formats.prompt = 'invalid'; data.metadata.raw = {}; data.metadata.prompt_graph = { status: 'limited', value: null }; data.metadata.record_formats.workflow = 'original-text'; data.metadata.raw.workflow = '{"nodes":[]}'; data.metadata.workflow_graph = { status: 'parsed', value: { nodes: [] } }; }, ]) { const data = envelope(); mutate(data); let extractions = 0; assert.throws(() => toRecord(data, image(), () => { extractions++; return extractGenerationFacts({}); }), TypeError); @@ -144,6 +146,85 @@ test('record rejects contradictory omission provenance before extracting facts', } }); +function limitedWorkflow(data = envelope()) { + data.metadata.record_formats.workflow = 'limited'; + data.metadata.workflow_graph = { status: 'limited', value: null }; + return data; +} + +test('record accepts limited workflow format without raw and with a limited graph', () => { + const toRecord = contract('toInspectionRecord'); + const record = toRecord(limitedWorkflow(), image(), extractGenerationFacts); + assert.deepEqual(record.metadata.record_formats, { prompt: 'reserialized-json', workflow: 'limited' }); + assert.deepEqual(record.metadata.raw, { prompt: '{"rounded":"raw copy"}' }); + assert.deepEqual(record.metadata.workflow_graph, { status: 'limited', value: null }); + assert.equal(record.metadata.status, 'available'); + assert.equal(record.generation.seed.value, '9007199254740993', 'a limited workflow never hides prompt-graph facts'); + const workflowOnly = limitedWorkflow(); + Object.assign(workflowOnly.metadata, { raw: {}, record_formats: { workflow: 'limited' }, prompt_graph: { status: 'absent', value: null } }); + const facts = toRecord(workflowOnly, image(), extractGenerationFacts).generation; + assert.ok(Object.values(facts).every(fact => fact.status === 'unresolved' && fact.reason === 'projection-limited')); + const omitted = limitedWorkflow(); + Object.assign(omitted.metadata, { raw: {}, parameters: null, status: 'limited', omitted_reason: 'response-size-limit', prompt_graph: { status: 'limited', value: null } }); + assert.deepEqual(toRecord(omitted, image(), extractGenerationFacts).metadata.record_formats, { prompt: 'reserialized-json', workflow: 'limited' }); +}); + +test('record rejects limited format with raw present', () => { + const toRecord = contract('toInspectionRecord'); + for (const mutate of [ + data => { data.metadata.raw.workflow = '{"nodes":[]}'; }, + data => { data.metadata.workflow_graph = { status: 'parsed', value: { nodes: [] } }; }, + data => { data.metadata.workflow_graph = { status: 'malformed', value: null }; }, + data => { data.metadata.workflow_graph = { status: 'absent', value: null }; }, + // Role records fail closed on the server: they are never limited. + data => { data.metadata.record_formats.prompt = 'limited'; delete data.metadata.raw.prompt; data.metadata.prompt_graph = { status: 'limited', value: null }; }, + data => { data.metadata.record_formats.parameters = 'limited'; }, + ]) { + const data = limitedWorkflow(); mutate(data); let extractions = 0; + assert.throws(() => toRecord(data, image(), () => { extractions++; return extractGenerationFacts({}); }), TypeError); + assert.equal(extractions, 0); + } +}); + +test('record accepts the workflow-first response omission of an invalid workflow record', () => { + const data = envelope(); + data.metadata.record_formats.workflow = 'invalid'; + data.metadata.workflow_graph = { status: 'limited', value: null }; + const record = contract('toInspectionRecord')(data, image(), extractGenerationFacts); + assert.deepEqual(record.metadata.prompt_graph.status, 'parsed', 'prompt evidence outlives workflow evidence'); + assert.equal(record.generation.seed.value, '9007199254740993'); +}); + +test('record accepts closed reason codes and rejects contradictory ones', () => { + const toRecord = contract('toInspectionRecord'); + const codes = ['work-limit', 'depth-limit', 'text-limit', 'graph-ceiling', 'unsupported-node', 'runtime-value', 'composite', 'no-sampler', 'malformed', 'multistage-differs']; + for (const reason_code of codes) { + const data = envelope(); Object.assign(data.resolution, { state: 'unresolved', reason_code, node: null }); + assert.doesNotThrow(() => toRecord(data, image(), extractGenerationFacts), reason_code); + } + const node = envelope(); Object.assign(node.resolution, { state: 'unresolved', reason_code: 'unsupported-node', node: { id: '7', class_type: 'LoRAfromText' } }); + assert.doesNotThrow(() => toRecord(node, image(), extractGenerationFacts)); + node.resolution.node.class_type = null; // the server names a non-string class as null + assert.doesNotThrow(() => toRecord(node, image(), extractGenerationFacts)); + for (const state of ['resolved', 'absent', 'ambiguous', 'unresolved']) { + const data = envelope(); Object.assign(data.resolution, { state, reason_code: null, node: null }); + assert.doesNotThrow(() => toRecord(data, image(), extractGenerationFacts), state); + } + for (const change of [ + { state: 'unresolved', reason_code: 'guess' }, { state: 'unresolved', reason_code: 1 }, + { state: 'resolved', reason_code: 'work-limit' }, { state: 'absent', reason_code: 'no-sampler' }, + { state: 'unresolved', reason_code: null, node: { id: '7', class_type: 'X' } }, + { state: 'unresolved', reason_code: 'unsupported-node', node: { id: 7, class_type: 'X' } }, + { state: 'unresolved', reason_code: 'unsupported-node', node: { id: '7' } }, + { state: 'unresolved', reason_code: 'unsupported-node', node: { id: '7', class_type: 7 } }, + { state: 'unresolved', reason_code: 'unsupported-node', node: 'X' }, + ]) { + const data = envelope(); Object.assign(data.resolution, change); let extractions = 0; + assert.throws(() => toRecord(data, image(), () => { extractions++; return extractGenerationFacts({}); }), TypeError, JSON.stringify(change)); + assert.equal(extractions, 0); + } +}); + test('record rejects root path media and invalid versioned fields with retryable TypeError', () => { const toRecord = contract('toInspectionRecord'); for (const mutate of [ diff --git a/tests/test_api_prompt_inspector.py b/tests/test_api_prompt_inspector.py index ea6743c..052de9a 100644 --- a/tests/test_api_prompt_inspector.py +++ b/tests/test_api_prompt_inspector.py @@ -7,8 +7,10 @@ from PIL import Image, PngImagePlugin try: from test_api_images import ImageAPITestCase, make_png + import inspection_parity as parity except ImportError: from tests.test_api_images import ImageAPITestCase, make_png + from tests import inspection_parity as parity from utils.pair_execution import EXECUTION_KEY, record_pair_execution from database.pair_ingestion import ingest_resolved_pair @@ -311,7 +313,9 @@ class TestPromptInspector(ImageAPITestCase): cases = [base[:-1], base[:30]] index = base.index(b"tEXt") broken = bytearray(base) - broken[index + 8] ^= 1 + # Corrupt the retained record's body: its CRC is checked. (A corrupt + # keyword names an unretained chunk, which is skipped unchecked.) + broken[index + 4 + len(b"prompt\0") + 1] ^= 1 cases.append(bytes(broken)) for kind, payload in ( (b"tEXt", b"prompt\0{}"), @@ -713,17 +717,93 @@ class TestPromptInspector(ImageAPITestCase): self.assertLessEqual(len(await response.read()), 8 * 1024 * 1024) self.assertNotIn(str(self.comfy_dir), json.dumps(result)) - async def test_workflow_raw_uses_same_aggregate_budget_and_read(self): - from utils.prompt_inspector import MAX_METADATA_BYTES + async def test_workflow_has_its_own_budget_and_never_hides_prompts(self): + from utils.prompt_inspector import ROLE_METADATA_BYTES, WORKFLOW_METADATA_BYTES path = self.output_dir / "image.png" - png(path, prompt=json.dumps(graph()), workflow="x" * MAX_METADATA_BYTES) + # Over the role budget, within the workflow budget: fully readable. + workflow = "x" * ROLE_METADATA_BYTES + png(path, prompt=json.dumps(graph()), workflow=workflow) response, result = await self.inspect() self.assertEqual(response.status, 200) - self.assertEqual(result.get("metadata", {}).get("status"), "unreadable") - self.assertEqual(result["prompts"]["positive"]["status"], "unresolved") - self.assertEqual(result["metadata"]["raw"], {}) - self.assertEqual(result["metadata"]["record_formats"], {}) + self.assertEqual(result["metadata"]["status"], "available") + self.assertEqual(result["prompts"]["positive"], self.side("portrait")) + self.assertEqual(result["metadata"]["raw"]["workflow"], workflow) + self.assertEqual(result["metadata"]["workflow_graph"]["status"], "malformed") + # Over the workflow budget: only the workflow is limited. + png( + path, + prompt=json.dumps(graph()), + workflow="x" * (WORKFLOW_METADATA_BYTES + 1), + ) + response, result = await self.inspect() + self.assertEqual(response.status, 200) + self.assertEqual(result["metadata"]["status"], "available") + self.assertEqual(result["prompts"]["positive"], self.side("portrait")) + self.assertEqual(result["prompts"]["negative"], self.side("blurry")) + self.assertEqual( + result["metadata"]["record_formats"], + {"prompt": "original-text", "workflow": "limited"}, + ) + self.assertEqual(set(result["metadata"]["raw"]), {"prompt"}) + self.assertEqual( + result["metadata"]["workflow_graph"], {"status": "limited", "value": None} + ) + self.assertEqual(result["metadata"]["prompt_graph"]["status"], "parsed") + + def parity_fixture(self, name): + path = self.output_dir / f"{name}.png" + parity.write_png(path, parity.build(name)) + return path + + async def test_inspect_route_resolves_506_node_graph(self): + # f03: 506 executable nodes. The default (ingestion) profile's 500-node + # gate reports "Resolution limit reached"; the inspection profile bounds + # work instead of graph size, so the route resolves the pair. + self.parity_fixture("f03") + response, result = await self.inspect("f03.png") + self.assertEqual(response.status, 200) + self.assertEqual( + result["prompts"], + {"positive": self.side(parity.POS), "negative": self.side(parity.NEG)}, + ) + self.assertEqual(result["resolution"]["state"], "resolved") + self.assertIsNone(result["resolution"]["reason_code"]) + self.assertIsNone(result["resolution"]["node"]) + self.assertEqual(result["metadata"]["prompt_graph"]["status"], "parsed") + + async def test_inspect_route_reports_no_sampler_instead_of_absent(self): + # f17: a BasicGuider-only prompt graph is a record, never "not recorded". + self.parity_fixture("f17") + response, result = await self.inspect("f17.png") + self.assertEqual(response.status, 200) + self.assertEqual( + {role: side["status"] for role, side in result["prompts"].items()}, + {"positive": "unresolved", "negative": "unresolved"}, + ) + self.assertEqual(result["resolution"]["state"], "unresolved") + self.assertEqual(result["resolution"]["reason_code"], "no-sampler") + self.assertEqual( + result["resolution"]["reason"], + "No supported sampler pair found (saw SamplerCustomAdvanced)", + ) + + async def test_inspect_route_exposes_reason_code_and_blocking_node(self): + self.parity_fixture("f06a") + _, result = await self.inspect("f06a.png") + resolution = result["resolution"] + self.assertEqual(resolution["state"], "unresolved") + self.assertEqual(resolution["reason_code"], "unsupported-node") + self.assertEqual(set(resolution["node"]), {"id", "class_type"}) + self.assertIsInstance(resolution["node"]["class_type"], str) + png(self.output_dir / "image.png", prompt="{broken") + _, result = await self.inspect() + self.assertEqual(result["resolution"]["reason_code"], "malformed") + make_png(self.output_dir / "image.png") + _, result = await self.inspect() + self.assertEqual(result["resolution"]["state"], "absent") + self.assertIsNone(result["resolution"]["reason_code"]) + self.assertIsNone(result["resolution"]["node"]) async def test_workflow_only_keeps_existing_role_uncertainty(self): path = png(self.output_dir / "image.png", workflow={"nodes": []}) @@ -988,7 +1068,6 @@ class TestPromptInspector(ImageAPITestCase): source = json.dumps(graph()) for suffix, expected in ( (',"duplicate":1,"duplicate":2', "malformed"), - (',"nonfinite":NaN', "malformed"), (',"depth":' + "[" * 64 + "0" + "]" * 64, "limited"), ): _, result = await self.video_inspect( @@ -1002,6 +1081,25 @@ class TestPromptInspector(ImageAPITestCase): self.assertIsNone(result["metadata"]["prompt_graph"]["value"]) self.assertEqual(result["metadata"]["workflow_graph"]["status"], expected) + async def test_video_carrier_nonfinite_outside_graph_inputs_is_tolerated(self): + # ComfyUI quirks: NaN in an unrelated carrier field, in ``is_changed`` and + # in ``_meta`` are validated and dropped; they never reach the projection. + source = graph() + source["3"]["is_changed"] = ["NAN_MARKER"] + source["3"]["_meta"] = {"title": "K", "weight": "NAN_MARKER"} + text = json.dumps(source).replace('"NAN_MARKER"', "NaN") + _, result = await self.video_inspect( + '{"prompt":' + text + ',"workflow":{"nodes":[]},"nonfinite":NaN}' + ) + self.assertEqual(result["prompts"]["positive"], self.side("portrait")) + prompt_graph = result["metadata"]["prompt_graph"] + self.assertEqual(prompt_graph["status"], "parsed") + self.assertEqual(set(prompt_graph["value"]["3"]), {"class_type", "inputs"}) + self.assertEqual( + result["metadata"]["workflow_graph"], + {"status": "parsed", "value": {"nodes": []}}, + ) + async def test_video_numeric_role_text_never_becomes_sendable(self): source = graph(positive=123) _, result = await self.video_inspect(json.dumps({"prompt": source})) diff --git a/tests/test_ingestion_resolver_profile.py b/tests/test_ingestion_resolver_profile.py new file mode 100644 index 0000000..bbc717c --- /dev/null +++ b/tests/test_ingestion_resolver_profile.py @@ -0,0 +1,84 @@ +"""Ingestion keeps the default resolver profile; only inspection routes moved (D1). + +The live monitor and the admin scan write pairs and links to the user's +database, so their resolver semantics stay byte-for-byte today's. f03 (506 +executable nodes) separates the profiles: the default profile's 500-node gate +leaves it unresolved, while the inspection profile resolves it. +""" + +import logging +import unittest +from unittest.mock import Mock, patch + +from utils import prompt_pair_resolver as resolver + +try: + from tests import inspection_parity as parity +except ImportError: # pragma: no cover - discover -s tests puts tests/ first + import inspection_parity as parity + +GATED = "Resolution limit reached" + + +def _f03_metadata(): + return {"prompt": parity.build("f03")["prompt"]} + + +class _Spy: + """Wrap the real resolver, recording each call's profile and outcome.""" + + def __init__(self): + self.calls = [] + + def __call__(self, metadata, **kwargs): + result = resolver.resolve_prompt_pair(metadata, **kwargs) + self.calls.append((kwargs.get("profile", resolver.DEFAULT_PROFILE), result)) + return result + + def assert_default_and_gated(self, case): + case.assertTrue(self.calls, "the ingestion path resolved the metadata") + for profile, result in self.calls: + case.assertIs(profile, resolver.DEFAULT_PROFILE) + case.assertEqual((result["state"], result["reason"]), ("unresolved", GATED)) + + +class TestIngestionKeepsDefaultProfile(unittest.TestCase): + def test_f03_separates_the_profiles(self): + metadata = _f03_metadata() + default = resolver.resolve_prompt_pair(metadata) + self.assertEqual((default["state"], default["reason"]), ("unresolved", GATED)) + inspection = resolver.resolve_prompt_pair( + metadata, profile=resolver.INSPECTION_PROFILE + ) + self.assertEqual(inspection["state"], "resolved") + + def test_live_monitor_lookup_and_linking_use_the_default_profile(self): + from utils.image_monitor import ImageGenerationHandler + + handler = ImageGenerationHandler.__new__(ImageGenerationHandler) + handler.db_manager = Mock() + handler.db_manager.get_prompt_by_hash.return_value = None + handler.logger = logging.getLogger("test.ingestion-profile") + spy, metadata = _Spy(), _f03_metadata() + with patch("utils.image_monitor.resolve_prompt_pair", spy): + handler._find_prompt_from_metadata(metadata) + handler.link_image_to_prompt("/out/f03.png", {"id": 1}, metadata) + self.assertEqual(len(spy.calls), 2) + spy.assert_default_and_gated(self) + handler.db_manager.link_image_to_prompt.assert_called_once() + + def test_admin_scan_ingestion_uses_the_default_profile(self): + from py.api import admin + + mixin = admin.AdminRoutesMixin.__new__(admin.AdminRoutesMixin) + mixin.db = Mock() + spy = _Spy() + with patch.object(admin, "resolve_prompt_pair", spy): + status = mixin._ingest_scanned_file("/out/f03.png", _f03_metadata()) + self.assertEqual(status, "unresolved") + spy.assert_default_and_gated(self) + mixin.db.assert_not_called() + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_inspection_corpus_tool.py b/tests/test_inspection_corpus_tool.py new file mode 100644 index 0000000..b4353f5 --- /dev/null +++ b/tests/test_inspection_corpus_tool.py @@ -0,0 +1,367 @@ +"""The read-only corpus tool (``tools/inspection_corpus.py``) on synthetic fixtures. + +The committed parity fixtures are copied into a disposable root. The tool must +reproduce the counts their ``vectors.json`` expectations imply, leave the root +byte-for-byte and mtime-for-mtime unchanged, refuse outputs inside the root or the +code tree, keep file names and prompt text out of its output, and compute H1 +against a baseline file. +""" + +import contextlib +import hashlib +import io +import json +import os +import shutil +import struct +import tempfile +import unittest +import unittest.mock +import zlib +from collections import Counter +from pathlib import Path + +from tools import inspection_corpus as corpus + +try: + from tests import inspection_parity as parity + from tests.open_handles import assert_closed +except ImportError: # pragma: no cover - discover -s tests puts tests/ first + import inspection_parity as parity + from open_handles import assert_closed + +FIXTURES = Path(parity.__file__).resolve().parent / "fixtures" / "inspection_parity" + + +def _snapshot(root: Path) -> dict: + state = {} + for directory, subdirectories, names in os.walk(root): + for name in subdirectories + names: + path = Path(directory, name) + info = path.lstat() + digest = ( + hashlib.sha256(path.read_bytes()).hexdigest() + if path.is_file() + else None + ) + state[path.relative_to(root).as_posix()] = ( + info.st_mtime_ns, + info.st_size, + digest, + ) + return state + + +def _expected(): + names = {path.stem for path in FIXTURES.glob("*.png")} + return [ + vector + for vector in parity.load_vectors() + if vector["profile"] == "inspection" + and vector["anchor"] is None + and vector["fixture"] in names + ] + + +def _chunk(kind: bytes, data: bytes) -> bytes: + crc = zlib.crc32(kind + data) & 0xFFFFFFFF + return struct.pack(">I", len(data)) + kind + data + struct.pack(">I", crc) + + +def _png(*texts: bytes) -> bytes: + header = _chunk(b"IHDR", struct.pack(">IIBBBBB", 1, 1, 8, 0, 0, 0, 0)) + body = b"".join(_chunk(b"tEXt", text) for text in texts) + return corpus.PNG_SIGNATURE + header + body + _chunk(b"IEND", b"") + + +class CorpusToolTest(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.addCleanup(assert_closed, self, temporary.name) + self.base = Path(temporary.name) + self.root = self.base / "root" + shutil.copytree(FIXTURES, self.root / "nested") + (self.root / "nested" / "vectors.json").unlink() + self.node = shutil.which("node") + if self.node is None and os.environ.get("CI"): + self.fail("Node.js is required in CI for corpus facts") + + def _run(self, *extra: str) -> tuple: + output = io.StringIO() + with contextlib.redirect_stdout(output), contextlib.redirect_stderr(output): + code = corpus.main(["--root", str(self.root), "--workers", "1", *extra]) + return code, output.getvalue() + + def _rows(self, path: Path) -> list: + return [json.loads(line) for line in path.read_text("utf-8").splitlines()] + + def test_corpus_tool_aggregates_synthetic_directory_read_only(self): + before = _snapshot(self.root) + out, summary = self.base / "out.jsonl", self.base / "summary.txt" + + code, printed = self._run("--out", str(out), "--summary", str(summary)) + + expected_code = corpus.EXIT_OK if self.node else corpus.EXIT_INVARIANT_FAILED + self.assertEqual(code, expected_code, printed) + self.assertEqual(_snapshot(self.root), before) + expected = _expected() + rows = self._rows(out) + self.assertEqual(len(rows), len(expected)) + report = corpus.build_report(rows, None, corpus._STATE.code.cap) + states = Counter(vector["expect"]["state"] for vector in expected) + self.assertEqual(report["states"], states) + codes = Counter( + str(vector["expect"]["reason_code"] or "null") for vector in expected + ) + self.assertEqual(report["reason_codes"], codes) + self.assertEqual(report["selection_kinds"], Counter({"null": len(expected)})) + self.assertEqual(report["harness_errors"], Counter()) + # An unreadable file is an inspector outcome, not a harness error. + self.assertEqual(report["metadata_status"]["unreadable"], 1) + if self.node is not None: + self.assertEqual(report["invariants"]["harness"]["status"], "PASS") + for name in ("H2", "H3", "H4", "H5", "H6", "H8"): + self.assertEqual(report["invariants"][name]["status"], "PASS", name) + self.assertEqual(report["invariants"]["H1"]["status"], "N/A") + if self.node is not None: + known = sum( + 1 + for vector in expected + if sum( + fact["status"] == "known" + for fact in (vector["expect"]["facts"] or {}).values() + ) + >= corpus.FACTS_KNOWN_TARGET + ) + self.assertEqual(report["facts_known_at_least_4"], known) + self.assertEqual(report["facts_errors"], Counter()) + self.assertEqual(summary.read_text("utf-8"), printed) + self.assertIn("\nstates: ", printed) + text = out.read_text("utf-8") + printed + for fixture in FIXTURES.glob("*.png"): + self.assertNotIn(fixture.name, text) + + def test_outputs_inside_root_or_code_tree_are_refused(self): + before = _snapshot(self.root) + inside = self.root / "nested" / "out.jsonl" + code_tree = corpus.TOOL_TREE / "corpus-out.jsonl" + cases = [ + ["--out", str(inside)], + ["--out", str(self.base / "ok.jsonl"), "--summary", str(code_tree)], + [ + "--out", + str(self.base / "same.txt"), + "--summary", + str(self.base / "same.txt"), + ], + ["--out", str(self.base)], + ] + link = self.base / "link" + try: + link.symlink_to(self.root, target_is_directory=True) + cases.append(["--out", str(link / "via-link.jsonl")]) + except OSError: # pragma: no cover - Windows without symlink privilege + pass + for arguments in cases: + with self.subTest(arguments=arguments): + code, printed = self._run(*arguments) + self.assertEqual(code, corpus.EXIT_REFUSED) + self.assertIn("inspection_corpus:", printed) + self.assertEqual(_snapshot(self.root), before) + self.assertFalse(code_tree.exists()) + self.assertFalse((self.base / "ok.jsonl").exists()) + + def test_h1_is_computed_against_a_baseline_file(self): + first = self.base / "first.jsonl" + self._run("--out", str(first), "--with-paths") + rows = self._rows(first) + self.assertTrue(all(row["path"].startswith("nested/") for row in rows)) + resolved = [row for row in rows if row["state"] == "resolved"] + unresolved = [row for row in rows if row["state"] == "unresolved"] + changed = {resolved[0]["id"], unresolved[0]["id"]} + baseline = [] + for row in rows: + if row["id"] == resolved[0]["id"]: + row = { + **row, + "prompts": {**row["prompts"], "positive": {"sha256": "0"}}, + } + if row["id"] == unresolved[0]["id"]: + row = {**row, "state": "resolved"} + baseline.append(row) + baseline_path = self.base / "baseline.jsonl" + baseline_path.write_text("".join(json.dumps(r) + "\n" for r in baseline)) + + code, printed = self._run( + "--out", str(self.base / "second.jsonl"), "--baseline", str(baseline_path) + ) + + self.assertEqual(code, corpus.EXIT_INVARIANT_FAILED, printed) + h1 = corpus.invariants( + self._rows(self.base / "second.jsonl"), baseline, corpus._STATE.code.cap + )["H1"] + self.assertEqual(h1["status"], "FAIL") + self.assertEqual(h1["count"], len(changed)) + self.assertEqual( + h1["breakdown"], + { + "resolved:texts-differ": 1, + f"unresolved:{unresolved[0]['reason_code']}": 1, + }, + ) + self.assertIn(" H1 FAIL 2", printed) + clean = corpus.invariants(rows, rows, corpus._STATE.code.cap)["H1"] + self.assertEqual((clean["status"], clean["count"]), ("PASS", 0)) + missing = corpus.invariants([], resolved, corpus._STATE.code.cap)["H1"] + self.assertEqual(missing["breakdown"], {"missing": len(resolved)}) + + def test_parallel_workers_match_the_single_process_rows(self): + single, parallel = self.base / "single.jsonl", self.base / "parallel.jsonl" + self._run("--out", str(single)) + self._run("--out", str(parallel), "--workers", "2") + + def stable(path): + rows = self._rows(path) + return sorted( + ({k: v for k, v in r.items() if k != "ms"} for r in rows), + key=lambda r: r["id"], + ) + + self.assertEqual(stable(parallel), stable(single)) + ids = [row["id"] for row in self._rows(parallel)] + self.assertEqual(ids, sorted(ids)) + + def test_symlinks_and_other_files_are_not_read(self): + outside = self.base / "outside.png" + shutil.copy(FIXTURES / "f02.png", outside) + (self.root / "notes.txt").write_text("not media") + try: + (self.root / "linked.png").symlink_to(outside) + except OSError: # pragma: no cover - Windows without symlink privilege + self.skipTest("symlinks unavailable") + extensions = frozenset({".png"}) + found = [rel for _, rel in corpus.media_files(self.root, extensions)] + self.assertEqual(len(found), len(_expected())) + self.assertNotIn("linked.png", found) + + def test_missing_root_baseline_or_code_tree_is_refused(self): + out = str(self.base / "x.jsonl") + for arguments in ( + ["--root", str(self.base / "absent"), "--out", out], + ["--root", str(self.root), "--out", out, "--baseline", out + ".none"], + ["--root", str(self.root), "--out", out, "--code", str(self.base)], + ): + with self.subTest(arguments=arguments): + with contextlib.redirect_stderr(io.StringIO()): + self.assertEqual(corpus.main(arguments), corpus.EXIT_REFUSED) + + def test_without_node_facts_fail_h3_and_the_run(self): + out = self.base / "out.jsonl" + code, printed = self._run("--out", str(out), "--node", "") + self.assertEqual(code, corpus.EXIT_INVARIANT_FAILED) + rows = self._rows(out) + self.assertTrue(all(row["facts_error"] == "node-unavailable" for row in rows)) + self.assertIn(f"facts errors: node-unavailable {len(rows)}", printed) + self.assertIn(f" H3 FAIL {len(rows)}", printed) + self.assertIn(f" harness FAIL {len(rows)}", printed) + + def test_a_node_that_exits_fails_the_run(self): + false = shutil.which("false") + if false is None: # pragma: no cover - Windows has no false binary + self.skipTest("no false binary") + code, printed = self._run("--out", str(self.base / "o.jsonl"), "--node", false) + self.assertEqual(code, corpus.EXIT_INVARIANT_FAILED) + self.assertIn("node-exited", printed) + self.assertIsNone(corpus._STATE.node) + + def test_a_missing_node_binary_is_refused(self): + missing = str(self.base / "no-such-node") + code, printed = self._run( + "--out", str(self.base / "o.jsonl"), "--node", missing + ) + self.assertEqual(code, corpus.EXIT_REFUSED) + self.assertIn("--node", printed) + + def test_harness_exceptions_fail_the_run(self): + out = self.base / "out.jsonl" + with unittest.mock.patch.object( + corpus, "_observe", side_effect=RuntimeError("boom") + ): + code, printed = self._run("--out", str(out)) + self.assertEqual(code, corpus.EXIT_INVARIANT_FAILED) + rows = self._rows(out) + self.assertEqual({row["error"] for row in rows}, {"RuntimeError"}) + self.assertIn(f" harness FAIL {len(rows)}", printed) + self.assertNotIn("boom", out.read_text("utf-8") + printed) + + +class CorpusHelpersTest(unittest.TestCase): + def test_malformed_cause_names_only_quirks_outside_inputs(self): + node = '{"1": {"class_type": "K", "inputs": {"seed": %s}%s}}' + cases = { + node % ("1", ', "is_changed": [NaN]'): "non-input", + node % ("NaN", ""): "input-nonfinite", + '{"1": {"inputs": {}}}': "shape", + "[1]": "shape", + '{"1": {}, "1": {}}': "unparseable", + "{": "unparseable", + } + for text, cause in cases.items(): + with self.subTest(text=text): + self.assertEqual(corpus.malformed_cause(text), cause) + + def test_png_duplicate_scan_compares_record_bytes(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.addCleanup(assert_closed, self, directory.name) + path = Path(directory.name, "f.png") + cases = ( + ((b"prompt\0{}", b"prompt\0{}"), True), + ((b"prompt\0{}", b"prompt\0[]"), False), + ((b"prompt\0{}", b"other\0x", b"other\0y"), None), + ) + for texts, expected in cases: + with self.subTest(texts=texts): + path.write_bytes(_png(*texts)) + self.assertIs(corpus.png_duplicates_identical(path), expected) + path.write_bytes(b"not a png") + self.assertIsNone(corpus.png_duplicates_identical(path)) + + def test_percentile_is_nearest_rank(self): + self.assertIsNone(corpus.percentile([], 0.5)) + self.assertEqual(corpus.percentile([3, 1, 2], 0.5), 2) + self.assertEqual(corpus.percentile(list(range(1, 101)), 0.99), 99) + + def test_h8_fails_on_slow_files_or_responses_over_the_cap(self): + fast = [{"id": "a", "ext": ".png", "ms": 1.0}] + self.assertEqual(corpus.invariants(fast, None, 8)["H8"]["status"], "PASS") + for rows in ( + [{"id": "a", "ext": ".png", "ms": corpus.MAX_MS}], + [{"id": "a", "ext": ".png", "ms": corpus.H8_FLOOR_MS + 1}], + [{"id": "a", "ext": ".png", "ms": 1.0, "over_cap": True}], + ): + with self.subTest(rows=rows): + result = corpus.invariants(rows, fast, 8)["H8"] + self.assertEqual(result["status"], "FAIL") + # A tiny baseline p99 cannot make H8 flap below the floor. + noisy = [{"id": "a", "ext": ".png", "ms": 3.0}] + self.assertEqual(corpus.invariants(noisy, fast, 8)["H8"]["status"], "PASS") + + def test_h5_excludes_negative_only_absence(self): + def row(positive, state): + return { + "ext": ".png", + "records": ["prompt"], + "state": state, + "prompts": {"positive": {"status": positive}}, + } + + rows = [row("resolved", "absent"), row("absent", "absent"), row("absent", "x")] + self.assertEqual(corpus.invariants(rows, None, 8)["H5"]["count"], 2) + no_record = {**row("absent", "absent"), "records": ["workflow"]} + self.assertEqual(corpus.invariants([no_record], None, 8)["H5"]["count"], 0) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_inspection_parity_facts.py b/tests/test_inspection_parity_facts.py new file mode 100644 index 0000000..469c1fa --- /dev/null +++ b/tests/test_inspection_parity_facts.py @@ -0,0 +1,112 @@ +"""Shared parity vectors: the workspace's JS facts match ``vectors.json``. + +Each ``inspection`` vector's file goes through the real current-file route +pipeline (``inspect_current_file`` then ``encode_inspection_response``). The +exact response text is handed to Node, where the workspace's own +``projectInspectionPayload`` validates it and ``extractGenerationFacts`` reads +it. Python and JS therefore assert the same files with the same expectations. +""" + +import json +import os +import shutil +import subprocess # nosec B404 +import tempfile +import unittest +from pathlib import Path + +from py.api.inspection_service import inspect_current_file +from utils.inspection_response import encode_inspection_response + +try: + from tests import inspection_parity as parity + from tests.open_handles import assert_closed +except ImportError: # pragma: no cover - discover -s tests puts tests/ first + import inspection_parity as parity + from open_handles import assert_closed + +HARNESS = ( + Path(__file__).resolve().parent / "js" / "support" / "inspection-parity-facts.mjs" +) +LIBRARY = { + "status": "none", + "scope": "saved-library", + "positive_id": None, + "pair_id": None, + "title": None, + "category": None, + "tags": [], + "metadata_scope": None, +} +NODE_TIMEOUT_SECONDS = 120 + + +def _encoded(name: str, path: Path) -> str: + observed = inspect_current_file( + path, source_label="parity", saved=None, library=LIBRARY + ) + body = encode_inspection_response( + image={"root_index": 0, "path": f"{name}.png", "filename": f"{name}.png"}, + root_scope="a" * 64, + file=observed["file"], + prompts=observed["prompts"], + resolution=observed["resolution"], + inspection=observed["inspection"], + library=observed["library"], + ) + return body.decode("utf-8") + + +def _responses(vectors, directory: Path) -> list: + responses = [] + for vector in vectors: + name = vector["fixture"] + if name in parity.TEST_TIME: + path = parity.fixture_path(name, directory) + parity.write_png(path, parity.build(name)) + else: + path = parity.fixture_path(name) + responses.append({"name": vector["name"], "body": _encoded(name, path)}) + return responses + + +class TestParityFacts(unittest.TestCase): + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.addCleanup(assert_closed, self, temporary.name) + self.directory = Path(temporary.name) + + def test_vectors_json_facts_match_workspace_js(self): + node = shutil.which("node") + if node is None: # pragma: no cover - CI runners and dev hosts ship Node + if os.environ.get("CI"): + self.fail("Node.js is required in CI for the facts vectors") + self.skipTest("Node.js is not installed") + vectors = [v for v in parity.load_vectors() if v["profile"] == "inspection"] + self.assertTrue(vectors) + payload = json.dumps(_responses(vectors, self.directory)) + # Fixed local Node harness, no shell. + completed = subprocess.run( # nosec B603 + [node, str(HARNESS)], + input=payload.encode("utf-8"), + capture_output=True, + timeout=NODE_TIMEOUT_SECONDS, + check=False, + ) + self.assertEqual(completed.returncode, 0, completed.stderr.decode("utf-8")) + observed = json.loads(completed.stdout) + for vector in vectors: + with self.subTest(vector=vector["name"]): + self.assertEqual(observed[vector["name"]], vector["expect"]["facts"]) + + def test_default_vectors_carry_no_facts(self): + # Facts are a workspace (inspection) concern; default vectors pin the + # ingestion resolver only. + for vector in parity.load_vectors(): + if vector["profile"] == "default": + self.assertIsNone(vector["expect"]["facts"], vector["name"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_inspection_parity_fixtures.py b/tests/test_inspection_parity_fixtures.py new file mode 100644 index 0000000..9d13dd5 --- /dev/null +++ b/tests/test_inspection_parity_fixtures.py @@ -0,0 +1,148 @@ +"""Committed synthetic inspection parity fixtures stay reproducible and referenced.""" + +import json +import tempfile +import unittest +from pathlib import Path + +try: + from tests import inspection_parity as parity +except ImportError: # pragma: no cover - discover -s tests puts tests/ first + import inspection_parity as parity + + +class InspectionParityFixtureTests(unittest.TestCase): + def test_committed_fixtures_match_builders(self): + # Decoded text chunks, not bytes: zlib output differs between platforms. + for name in parity.COMMITTED: + with self.subTest(fixture=name): + path = parity.fixture_path(name) + self.assertTrue(path.is_file(), f"missing committed fixture {path}") + self.assertLess(path.stat().st_size, parity.MAX_FIXTURE_BYTES) + self.assertEqual( + parity.read_text_chunks(path), + parity.expected_text_chunks(parity.build(name)), + ) + + def test_no_uncommitted_files_in_fixture_directory(self): + # Only fixture data counts; OS litter (.DS_Store, Thumbs.db) is ignored. + expected = {f"{name}.png" for name in parity.COMMITTED} | {"vectors.json"} + present = { + path.name + for path in parity.FIXTURE_DIR.iterdir() + if path.suffix in (".png", ".json") + } + self.assertEqual(present, expected) + + def test_writer_round_trips_extra_raw_chunks(self): + for name in ("f15", "f16"): + built = parity.build(name) + with tempfile.TemporaryDirectory() as directory: + path = Path(directory) / f"{name}.png" + parity.write_png(path, built) + chunks = parity.read_text_chunks(path) + with self.subTest(fixture=name): + self.assertEqual(chunks, parity.expected_text_chunks(built)) + self.assertGreater(len(chunks), 2) + + def test_duplicate_fixtures_have_the_intended_shape(self): + identical = parity.read_text_chunks(parity.fixture_path("f15")) + self.assertEqual(len(identical), 8) + self.assertEqual(len(set(identical)), 2) + conflicting = [ + text + for key, text in parity.read_text_chunks(parity.fixture_path("f16")) + if key == "prompt" + ] + self.assertEqual(len(conflicting), 2) + self.assertNotEqual(conflicting[0], conflicting[1]) + + def test_nan_fixture_keeps_comfyui_spelling_outside_inputs(self): + prompt = dict(parity.read_text_chunks(parity.fixture_path("f14")))["prompt"] + self.assertIn('"is_changed": [NaN]', prompt) + self.assertIn('"_meta": {"title": "CheckpointLoaderSimple"}', prompt) + for node in json.loads(prompt).values(): + for value in node["inputs"].values(): + self.assertEqual(value, value, "NaN stays outside inputs") + + def test_builders_are_deterministic_and_fresh(self): + for name in parity.COMMITTED + parity.TEST_TIME: + with self.subTest(fixture=name): + first, second = parity.build(name), parity.build(name) + self.assertEqual( + parity.expected_text_chunks(first), + parity.expected_text_chunks(second), + ) + if first["prompt"] is not None: + first["prompt"].clear() + self.assertTrue(parity.build(name)["prompt"]) + with self.assertRaises(KeyError): + parity.build("f99") + + def test_test_time_shapes_exceed_commit_size(self): + sizes = { + name: sum( + len(text) for _, text in parity.expected_text_chunks(parity.build(name)) + ) + for name in parity.TEST_TIME + } + self.assertGreater(sizes["f01"], 3 * parity.MAX_FIXTURE_BYTES) + prompt = parity.build("f01")["prompt"] + self.assertEqual(len(prompt), 688) + self.assertEqual(sum(":" in node_id for node_id in prompt), 600) + self.assertGreater(sizes["f04"], 2_500_000) + self.assertGreater(sizes["f05"], 900_000) + + def test_vectors_reference_existing_fixtures(self): + vectors = parity.load_vectors() + self.assertTrue(vectors) + names = [vector["name"] for vector in vectors] + self.assertEqual(len(names), len(set(names)), "vector names are unique") + known = set(parity.COMMITTED) | set(parity.TEST_TIME) + for vector in vectors: + with self.subTest(vector=vector["name"]): + self.assertIn(vector["fixture"], known) + if vector["fixture"] in parity.COMMITTED: + self.assertTrue(parity.fixture_path(vector["fixture"]).is_file()) + self.assertIn(vector["profile"], ("default", "inspection")) + self.assertEqual(set(vector["expect"]), set(parity.EXPECT_KEYS)) + facts = vector["expect"]["facts"] + if vector["profile"] == "default": + # Facts are a property of the inspection response only. + self.assertIsNone(facts) + else: + self.assertEqual(set(facts), set(parity.FACT_FIELDS)) + for fact in facts.values(): + self.assertEqual(set(fact), {"status", "value"}) + for profile in ("inspection", "default"): + covered = { + vector["fixture"] for vector in vectors if vector["profile"] == profile + } + self.assertEqual( + covered, set(parity.COMMITTED) | set(parity.TEST_TIME), profile + ) + + def test_vectors_match_current_resolution(self): + """Baseline vectors pin today's outcomes; behavior changes edit them.""" + for vector in parity.load_vectors(): + with self.subTest(vector=vector["name"]): + expected = dict(vector["expect"]) + expected.pop("facts") + self.assertEqual(parity.observe(vector), expected) + + def test_observe_builds_test_time_fixtures_instead_of_reading_missing_files(self): + # A missing file would otherwise read as "unreadable" and hide a stale vector. + for name in parity.TEST_TIME: + with self.subTest(fixture=name): + self.assertFalse(parity.fixture_path(name).exists()) + observed = parity.observe( + {"fixture": name, "profile": "default", "anchor": None} + ) + self.assertEqual(observed["metadata_status"], "available") + self.assertNotEqual(observed["state"], "absent") + with self.assertRaises(FileNotFoundError): + parity.observe({"fixture": "f02", "profile": "inspection"}, "missing-dir") + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_inspection_response.py b/tests/test_inspection_response.py index 79c81d6..382dadb 100644 --- a/tests/test_inspection_response.py +++ b/tests/test_inspection_response.py @@ -2,11 +2,15 @@ import io import json +import struct import tempfile import unittest +import zlib from pathlib import Path from unittest.mock import MagicMock, patch +from PIL import Image + from utils.inspection_response import encode_inspection_response from utils.prompt_inspector import ( InspectionRead, @@ -18,6 +22,13 @@ from utils.prompt_inspector import ( ) from utils.pair_execution import EXECUTION_KEY +try: + from tests import inspection_parity as parity + from tests.open_handles import assert_closed +except ImportError: # pragma: no cover - discover -s tests puts tests/ first + import inspection_parity as parity + from open_handles import assert_closed + CAP = 8 * 1024 * 1024 IMAGE = {"root_index": 0, "path": "image.png", "filename": "image.png"} PROMPTS = { @@ -134,15 +145,23 @@ class TestInspectionResponse(unittest.TestCase): self.assertEqual(result["metadata"]["raw"], read.metadata) self.assertEqual(result["prompts"], PROMPTS) - def test_graph_node_limit_counts_nodes_not_json_fields(self): - for count, status in ((500, "parsed"), (501, "limited")): + def test_graph_ceiling_is_limited(self): + # f27: the 20,000-node backstop, built at test time. Prompt nodes count + # top-level API nodes; workflow nodes include subgraph definitions. + for count, status in ((20_000, "parsed"), (20_001, "limited")): prompt = graph_text(count) + half = count // 2 workflow = json.dumps( { "nodes": [ {"id": i, "type": "Primitive", "widgets_values": [1, True]} - for i in range(count) - ] + for i in range(count - half) + ], + "definitions": { + "subgraphs": [ + {"id": "sg", "nodes": [{"id": i} for i in range(half)]} + ] + }, } ) for read in ( @@ -162,12 +181,237 @@ class TestInspectionResponse(unittest.TestCase): ): result = json.loads(encode(read)) for key in ("prompt_graph", "workflow_graph"): - self.assertEqual(result["metadata"][key]["status"], status) - if status == "limited": - self.assertIsNone(result["metadata"][key]["value"]) + with self.subTest(count=count, key=key): + self.assertEqual(result["metadata"][key]["status"], status) + if status == "limited": + self.assertIsNone(result["metadata"][key]["value"]) self.assertEqual(result["metadata"]["raw"], read.metadata) self.assertEqual(result["prompts"], PROMPTS) + def test_prompt_graph_with_600_executable_nodes_is_parsed(self): + graph = parity.build("f03")["prompt"] + for index in range(100): + graph[f"{700 + index}"] = { + "class_type": "PrimitiveInt", + "inputs": {"value": index}, + } + self.assertGreater(len(graph), 600) + text = json.dumps(graph) + result = json.loads(encode(inspection({"prompt": text}))) + projected = result["metadata"]["prompt_graph"] + self.assertEqual(projected["status"], "parsed") + self.assertEqual(len(projected["value"]), len(graph)) + self.assertEqual(projected["value"]["3"]["inputs"]["seed"], 42) + self.assertEqual(projected["value"]["3"]["inputs"]["cfg"], "6.5") + self.assertEqual(result["metadata"]["raw"]["prompt"], text) + + def test_workflow_with_513_ui_nodes_is_parsed_and_downloadable(self): + path = parity.fixture_path("f02") + chunks = dict(parity.read_text_chunks(path)) + self.assertGreater(len(json.loads(chunks["workflow"])["nodes"]), 513) + result = json.loads(encode(read_inspection_file(path))) + metadata = result["metadata"] + self.assertEqual(metadata["workflow_graph"]["status"], "parsed") + self.assertEqual( + metadata["workflow_graph"]["value"]["nodes"], + json.loads(chunks["workflow"])["nodes"], + ) + self.assertEqual(metadata["prompt_graph"]["status"], "parsed") + # The download is the original record text, byte for byte. + self.assertEqual(metadata["raw"]["workflow"], chunks["workflow"]) + self.assertEqual(metadata["record_formats"]["workflow"], "original-text") + + def test_nan_is_changed_and_meta_keep_prompt_graph_parsed(self): + path = parity.fixture_path("f14") + chunks = dict(parity.read_text_chunks(path)) + self.assertIn("NaN", chunks["prompt"]) + source = parity.build("f14")["prompt"] + result = json.loads(encode(read_inspection_file(path))) + metadata = result["metadata"] + graph = metadata["prompt_graph"] + self.assertEqual(graph["status"], "parsed") + self.assertEqual(set(graph["value"]), set(source)) + for node_id, node in graph["value"].items(): + self.assertEqual(set(node), {"class_type", "inputs"}) + self.assertEqual(node["class_type"], source[node_id]["class_type"]) + inputs = graph["value"]["3"]["inputs"] + expected = dict(source["3"]["inputs"], cfg="6.5", denoise="1.0") + self.assertEqual(inputs, expected) + self.assertEqual(graph["value"]["4"]["inputs"], source["4"]["inputs"]) + self.assertNotIn("_meta", json.dumps(graph)) + self.assertNotIn("is_changed", json.dumps(graph)) + # Raw records are never modified: the NaN stays in the download. + self.assertEqual(metadata["raw"]["prompt"], chunks["prompt"]) + self.assertEqual(metadata["workflow_graph"]["status"], "parsed") + + def test_nonfinite_inside_inputs_projects_null(self): + prompt = ( + '{"3":{"class_type":"KSampler","inputs":{"cfg":NaN,"seed":1,' + '"denoise":-Infinity,"list":[Infinity,0.5]},"is_changed":[NaN],' + '"_meta":{"title":"K","weight":Infinity}},"9":{"value":NaN}}' + ) + workflow = ( + '{"nodes":[{"id":1,"widgets_values":[NaN,1.5,"x"],' + '"extra":{"v":Infinity}}],"extra":{"ds":{"scale":-Infinity}}}' + ) + for read in ( + inspection({"prompt": prompt, "workflow": workflow}), + inspection( + {"prompt": prompt, "workflow": workflow}, + formats={ + "prompt": "reserialized-json", + "workflow": "reserialized-json", + }, + carrier='{"prompt":' + + prompt + + ',"workflow":' + + json.dumps(workflow) + + ',"other":{"x":NaN}}', + ), + ): + result = json.loads(encode(read)) + graph = result["metadata"]["prompt_graph"] + self.assertEqual(graph["status"], "parsed") + self.assertEqual( + graph["value"], + { + "3": { + "class_type": "KSampler", + "inputs": { + "cfg": None, + "seed": 1, + "denoise": None, + "list": [None, "0.5"], + }, + }, + "9": {}, + }, + ) + flow = result["metadata"]["workflow_graph"] + self.assertEqual(flow["status"], "parsed") + self.assertEqual( + flow["value"], + { + "nodes": [ + { + "id": 1, + "widgets_values": [None, "1.5", "x"], + "extra": {"v": None}, + } + ], + "extra": {"ds": {"scale": None}}, + }, + ) + self.assertEqual(result["metadata"]["raw"], read.metadata) + self.assertEqual(result["prompts"], PROMPTS) + + def test_prompt_projection_keeps_odd_node_shapes_and_rejects_non_objects(self): + prompt = '{"1":[NaN,2],"2":"text","3":{"inputs":{"a":1}},"4":{"_meta":{}}}' + expected = {"1": [None, 2], "2": "text", "3": {"inputs": {"a": 1}}, "4": {}} + carrier_cases = ( + ('{"prompt":' + prompt + "}", "parsed"), + ('{"prompt":[1],"workflow":{"nodes":[]}}', "malformed"), + ('{"prompt":' + json.dumps(prompt) + "}", "parsed"), + ("[]", "malformed"), + ) + result = json.loads(encode(inspection({"prompt": prompt}))) + self.assertEqual( + result["metadata"]["prompt_graph"], {"status": "parsed", "value": expected} + ) + for carrier, status in carrier_cases: + with self.subTest(carrier=carrier): + result = json.loads( + encode( + inspection( + {"prompt": prompt}, + formats={"prompt": "reserialized-json"}, + carrier=carrier, + ) + ) + ) + graph = result["metadata"]["prompt_graph"] + self.assertEqual(graph["status"], status) + self.assertEqual( + graph["value"], expected if status == "parsed" else None + ) + + def test_exact_numbers_survive_tolerance(self): + prompt = ( + '{"3":{"class_type":"KSampler","inputs":{"seed":1152921504606846976,' + '"steps":1e3,"cfg":0.1,"denoise":1e0,"exact":6.5000000000000000001,' + '"negative_seed":-1152921504606846976,"safe":9007199254740991},' + '"is_changed":[NaN]}}' + ) + workflow = ( + '{"nodes":[{"id":1,"widgets_values":[1152921504606846976,1e3,0.1,NaN]}]}' + ) + result = json.loads( + encode(inspection({"prompt": prompt, "workflow": workflow})) + ) + for key in ("prompt_graph", "workflow_graph"): + self.assertEqual(result["metadata"][key]["status"], "parsed") + inputs = result["metadata"]["prompt_graph"]["value"]["3"]["inputs"] + self.assertEqual( + inputs, + { + "seed": str(2**60), + "steps": "1E+3", + "cfg": "0.1", + "denoise": "1", + "exact": "6.5000000000000000001", + "negative_seed": str(-(2**60)), + "safe": 9007199254740991, + }, + ) + self.assertEqual(inputs["seed"], "1152921504606846976") + widgets = result["metadata"]["workflow_graph"]["value"]["nodes"][0] + self.assertEqual( + widgets["widgets_values"], ["1152921504606846976", "1E+3", "0.1", None] + ) + + def test_duplicate_json_keys_still_malformed(self): + for prompt, workflow in ( + ('{"1":{},"1":{}}', '{"nodes":[],"nodes":[]}'), + ( + '{"1":{"class_type":"A","inputs":{},"_meta":{"t":1,"t":2}}}', + '{"nodes":[{"id":1,"extra":{"a":NaN,"a":NaN}}]}', + ), + ( + '{"1":{"class_type":"A","inputs":{"x":NaN,"x":1}}}', + '{"nodes":[{"id":1,"id":2}]}', + ), + ): + with self.subTest(prompt=prompt): + result = json.loads( + encode(inspection({"prompt": prompt, "workflow": workflow})) + ) + for key in ("prompt_graph", "workflow_graph"): + self.assertEqual( + result["metadata"][key], {"status": "malformed", "value": None} + ) + self.assertEqual( + result["metadata"]["raw"], {"prompt": prompt, "workflow": workflow} + ) + valid = '{"1":{"class_type":"A","inputs":{}}}' + for carrier in ( + '{"prompt":' + valid + ',"prompt":' + valid + "}", + '{"prompt":' + valid + ',"other":{"a":1,"a":2}}', + ): + with self.subTest(carrier=carrier): + result = json.loads( + encode( + inspection( + {"prompt": valid}, + formats={"prompt": "reserialized-json"}, + carrier=carrier, + ) + ) + ) + self.assertEqual( + result["metadata"]["prompt_graph"], + {"status": "malformed", "value": None}, + ) + def test_strict_projection_depth_is_separate_from_semantic_link_depth(self): for depth in (64, 65): # API map root + node + inputs + nested array containers. @@ -238,8 +482,6 @@ class TestInspectionResponse(unittest.TestCase): def test_strict_malformed_records_preserve_original_raw_and_roles(self): for text in ( '{"1":{},"1":{}}', - '{"1":{"value":NaN}}', - '{"1":{"value":Infinity}}', "{broken", "[]", "42", @@ -318,6 +560,42 @@ class TestInspectionResponse(unittest.TestCase): result["metadata"]["omitted_reason"], "response-size-limit" ) + def test_response_budget_drops_workflow_graph_before_prompt_graph(self): + import utils.inspection_response as response + + with tempfile.TemporaryDirectory() as directory: + path = parity.fixture_path("f05", directory) + parity.write_png(path, parity.build("f05")) + read = read_inspection_file(path) + + def metadata_at(cap, read): + with patch.object(response, "MAX_RESPONSE_BYTES", cap): + body = encode(read) + self.assertLessEqual(len(body), cap) + return json.loads(body)["metadata"] + + # Step 1: the workflow graph alone is dropped; prompt evidence stays. + step1 = metadata_at(4_500_000, read) + self.assertEqual(step1["workflow_graph"], {"status": "limited", "value": None}) + self.assertEqual(step1["prompt_graph"]["status"], "parsed") + self.assertEqual(step1["status"], "available") + self.assertIn("workflow", step1["raw"]) + + # Step 2: a large prompt graph goes too, while raw records stay. + large = inspection({"prompt": graph_text(20000)}) + step2 = metadata_at(3_000_000, large) + self.assertEqual(step2["prompt_graph"], {"status": "limited", "value": None}) + self.assertEqual(step2["workflow_graph"]["status"], "absent") + self.assertEqual(step2["status"], "available") + self.assertIn("prompt", step2["raw"]) + + # Step 3: raw is omitted atomically. + step3 = metadata_at(3_000_000, read) + self.assertEqual(step3["prompt_graph"]["status"], "limited") + self.assertEqual(step3["workflow_graph"]["status"], "limited") + self.assertEqual(step3["raw"], {}) + self.assertEqual(step3["omitted_reason"], "response-size-limit") + def test_response_byte_boundary_is_inclusive(self): prompts = { role: { @@ -505,3 +783,483 @@ class TestInspectionResponse(unittest.TestCase): self.assertEqual(body["library"]["status"], "limited") self.assertNotIn("image", body) self.assertNotIn("root_scope", body) + + +MIB = 1024 * 1024 + + +def _text_chunk(kind, keyword, body): + return parity._raw_chunk(kind, keyword.encode("latin-1") + b"\0" + body) + + +def _png_with_chunks(path, chunks): + """A tiny real PNG with raw metadata chunks spliced before IEND, then closed.""" + buffer = io.BytesIO() + with Image.new("RGB", (4, 4), (40, 80, 120)) as image: + image.save(buffer, format="PNG") + data = buffer.getvalue() + iend = data.rindex(b"IEND") - 4 + with open(path, "wb") as stream: + stream.write(data[:iend]) + for chunk in chunks: + stream.write(chunk) + stream.write(data[iend:]) + return path + + +class TestPerRecordBudgets(unittest.TestCase): + """Role records keep the 2 MiB budget; a workflow has its own 4 MiB budget.""" + + def setUp(self): + temporary = tempfile.TemporaryDirectory() + self.addCleanup(temporary.cleanup) + self.addCleanup(assert_closed, self, temporary.name) + self.directory = Path(temporary.name) + + def prompt_chunk(self): + prompt = json.dumps(parity.build("f04")["prompt"]).encode("latin-1") + return _text_chunk(b"tEXt", "prompt", prompt) + + def assert_prompts_resolved(self, read): + from utils.prompt_inspector import inspect_prompt_evidence + from utils.prompt_pair_resolver import INSPECTION_PROFILE + + evidence = inspect_prompt_evidence(read.metadata, profile=INSPECTION_PROFILE) + self.assertEqual(evidence["resolution"]["state"], "resolved") + self.assertEqual(evidence["prompts"]["positive"]["text"], parity.POS) + self.assertEqual(evidence["prompts"]["negative"]["text"], parity.NEG) + + def test_budget_constants(self): + from utils import prompt_inspector + + self.assertEqual(prompt_inspector.ROLE_METADATA_BYTES, 2 * MIB) + self.assertEqual(prompt_inspector.WORKFLOW_METADATA_BYTES, 4 * MIB) + self.assertEqual(prompt_inspector.MAX_KEYWORD_BYTES, 80) + self.assertEqual(MAX_METADATA_BYTES, prompt_inspector.ROLE_METADATA_BYTES) + default = inspection({"prompt": "{}"}) + self.assertEqual(default.limited_records, frozenset()) + + def test_f04_large_workflow_within_budget_no_longer_hides_prompts(self): + # f04's 2.5 MB workflow is over the old shared 2 MiB budget but within + # the 4 MiB workflow budget: everything is readable now. + path = self.directory / "f04.png" + parity.write_png(path, parity.build("f04")) + read = read_inspection_file(path) + self.assert_prompts_resolved(read) + metadata = json.loads(encode(read))["metadata"] + self.assertEqual(metadata["status"], "available") + self.assertEqual(metadata["record_formats"]["workflow"], "original-text") + self.assertEqual(metadata["workflow_graph"]["status"], "parsed") + self.assertEqual(metadata["prompt_graph"]["status"], "parsed") + + def test_oversize_workflow_chunk_keeps_prompt_evidence(self): + from utils.prompt_inspector import WORKFLOW_METADATA_BYTES + + built = parity.build("f04") # f04's shape, its note grown past budget + built["workflow"]["nodes"][-1]["widgets_values"] = [ + "A" * (WORKFLOW_METADATA_BYTES + 1) + ] + path = self.directory / "f04-oversize.png" + parity.write_png(path, built) + read = read_inspection_file(path) + self.assertNotIn("workflow", read.metadata) + self.assertEqual(read.limited_records, frozenset({"workflow"})) + self.assert_prompts_resolved(read) + metadata = json.loads(encode(read))["metadata"] + self.assertEqual(metadata["status"], "available") + self.assertEqual( + metadata["workflow_graph"], {"status": "limited", "value": None} + ) + self.assertEqual(metadata["prompt_graph"]["status"], "parsed") + self.assertEqual(metadata["record_formats"]["workflow"], "limited") + self.assertEqual(metadata["record_formats"]["prompt"], "original-text") + self.assertNotIn("workflow", metadata["raw"]) + self.assertIn("prompt", metadata["raw"]) + + def test_oversize_workflow_is_skipped_by_seek_not_read(self): + from utils.prompt_inspector import WORKFLOW_METADATA_BYTES + + workflow = b"A" * (WORKFLOW_METADATA_BYTES + 1) + path = _png_with_chunks( + self.directory / "seek.png", + [_text_chunk(b"tEXt", "workflow", workflow), self.prompt_chunk()], + ) + reads = self.tracked_reads(path) + self.assertLess(sum(reads), 64 * 1024) + + def tracked_reads(self, path): + original_open = type(path).open + reads = [] + + def tracked_open(item, *args, **kwargs): + stream = original_open(item, *args, **kwargs) + original_read = stream.read + + def read(size=-1): + value = original_read(size) + reads.append(len(value)) + return value + + stream.read = read + return stream + + with patch.object(type(path), "open", tracked_open): + read = read_inspection_file(path) + self.assert_prompts_resolved(read) + return reads + + def test_oversize_compressed_workflow_stops_inflating(self): + from utils import prompt_inspector + + workflow = json.dumps({"nodes": [], "note": "A" * (6 * MIB)}).encode() + compressed = zlib.compress(workflow, 9) + self.assertLess(len(compressed), prompt_inspector.WORKFLOW_METADATA_BYTES) + produced = [] + real = zlib.decompressobj + + class Tracked: + def __init__(self): + self.inner = real() + + def decompress(self, data, max_length=0): + self.assert_bounded(max_length) + value = self.inner.decompress(data, max_length) + produced.append(len(value)) + return value + + @staticmethod + def assert_bounded(max_length): + if not 0 < max_length <= prompt_inspector.WORKFLOW_METADATA_BYTES + 1: + raise AssertionError(f"unbounded inflate: {max_length}") + + def __getattr__(self, name): + return getattr(self.inner, name) + + for kind, body in ( + (b"zTXt", b"\0" + compressed), + (b"iTXt", b"\1\0\0\0" + compressed), + ): + with self.subTest(kind=kind): + produced.clear() + path = _png_with_chunks( + self.directory / "compressed.png", + [self.prompt_chunk(), _text_chunk(kind, "workflow", body)], + ) + with patch.object(prompt_inspector.zlib, "decompressobj", Tracked): + read = read_inspection_file(path) + self.assertTrue(produced) + self.assertLessEqual( + max(produced), prompt_inspector.WORKFLOW_METADATA_BYTES + 1 + ) + self.assertEqual(read.limited_records, frozenset({"workflow"})) + self.assertEqual(read.record_formats["workflow"], "limited") + self.assertNotIn("workflow", read.metadata) + self.assert_prompts_resolved(read) + + def test_compressed_workflow_over_budget_is_skipped(self): + from utils.prompt_inspector import WORKFLOW_METADATA_BYTES + + body = b"\0" + bytes(range(256)) * (WORKFLOW_METADATA_BYTES // 256 + 1) + path = _png_with_chunks( + self.directory / "compressed-length.png", + [_text_chunk(b"zTXt", "workflow", body), self.prompt_chunk()], + ) + read = read_inspection_file(path) + self.assertEqual(read.limited_records, frozenset({"workflow"})) + self.assert_prompts_resolved(read) + + def test_role_records_over_budget_still_unreadable(self): + from utils.prompt_inspector import ROLE_METADATA_BYTES + + prompt = self.prompt_chunk() + cases = { + "prompt-text": [ + _text_chunk(b"tEXt", "prompt", b"x" * (ROLE_METADATA_BYTES + 1)) + ], + "prompt-inflate": [ + _text_chunk( + b"zTXt", + "prompt", + b"\0" + zlib.compress(b"x" * (ROLE_METADATA_BYTES + 1)), + ) + ], + "roles-together": [ + prompt, + _text_chunk(b"tEXt", "parameters", b"x" * ROLE_METADATA_BYTES), + ], + "execution-record": [ + prompt, + _text_chunk(b"tEXt", EXECUTION_KEY, b"x" * (ROLE_METADATA_BYTES + 1)), + ], + } + for name, chunks in cases.items(): + with self.subTest(case=name): + path = _png_with_chunks(self.directory / f"{name}.png", chunks) + with self.assertRaisesRegex( + MetadataReadError, "exceeds inspection limits" + ): + read_inspection_file(path) + + def test_role_budget_is_not_shared_with_workflow(self): + from utils.prompt_inspector import ROLE_METADATA_BYTES + + path = _png_with_chunks( + self.directory / "shared.png", + [ + self.prompt_chunk(), + _text_chunk(b"tEXt", "parameters", b"p" * (ROLE_METADATA_BYTES - 4096)), + _text_chunk(b"tEXt", "workflow", b"w" * (3 * MIB)), + ], + ) + read = read_inspection_file(path) + self.assertEqual(read.limited_records, frozenset()) + self.assertEqual(len(read.metadata["workflow"]), 3 * MIB) + self.assert_prompts_resolved(read) + + def test_identical_duplicate_chunks_are_accepted(self): + path = parity.fixture_path("f15") + chunks = dict(parity.read_text_chunks(path)) + read = read_inspection_file(path) + self.assertEqual(read.metadata["prompt"], chunks["prompt"]) + self.assertEqual(read.metadata["workflow"], chunks["workflow"]) + self.assert_prompts_resolved(read) + metadata = json.loads(encode(read))["metadata"] + self.assertEqual(metadata["prompt_graph"]["status"], "parsed") + self.assertEqual(metadata["workflow_graph"]["status"], "parsed") + + def test_conflicting_duplicate_chunks_fail(self): + with self.assertRaisesRegex(MetadataReadError, "Duplicate prompt metadata"): + read_inspection_file(parity.fixture_path("f16")) + prompt = json.dumps(parity.build("f04")["prompt"]).encode("latin-1") + cases = { + # Same decoded text, different bytes on disk: not byte-identical. + "kind": [ + self.prompt_chunk(), + _text_chunk(b"zTXt", "prompt", b"\0" + zlib.compress(prompt)), + ], + "length": [ + self.prompt_chunk(), + _text_chunk(b"tEXt", "prompt", prompt + b" "), + ], + "same-length": [ + self.prompt_chunk(), + _text_chunk(b"tEXt", "prompt", prompt.replace(b"red fox", b"red cat")), + ], + "workflow": [ + self.prompt_chunk(), + _text_chunk(b"tEXt", "workflow", b'{"nodes": []}'), + _text_chunk(b"tEXt", "workflow", b'{"nodes": [1]}'), + ], + } + for name, chunks in cases.items(): + with self.subTest(case=name): + path = _png_with_chunks(self.directory / f"{name}.png", chunks) + with self.assertRaisesRegex( + MetadataReadError, "Duplicate prompt metadata" + ): + read_inspection_file(path) + + def test_duplicate_of_limited_workflow_stays_limited(self): + from utils.prompt_inspector import WORKFLOW_METADATA_BYTES + + oversize = _text_chunk( + b"tEXt", "workflow", b"A" * (WORKFLOW_METADATA_BYTES + 1) + ) + path = _png_with_chunks( + self.directory / "limited-twice.png", + [oversize, self.prompt_chunk(), oversize], + ) + read = read_inspection_file(path) + self.assertEqual(read.limited_records, frozenset({"workflow"})) + self.assertNotIn("workflow", read.metadata) + self.assert_prompts_resolved(read) + + def test_identical_duplicates_do_not_consume_role_budget(self): + from utils.prompt_inspector import ROLE_METADATA_BYTES + + parameters = _text_chunk( + b"tEXt", "parameters", b"p" * (ROLE_METADATA_BYTES * 3 // 4) + ) + path = _png_with_chunks( + self.directory / "repeat.png", + [self.prompt_chunk(), parameters, parameters], + ) + read = read_inspection_file(path) + self.assertEqual(len(read.metadata["parameters"]), ROLE_METADATA_BYTES * 3 // 4) + + def test_workflow_duplicates_are_limited_without_touching_role_records(self): + from utils import prompt_inspector + + self.assertEqual(prompt_inspector.DUPLICATE_WORKFLOW_BYTES, 8 * MIB) + self.assertEqual(prompt_inspector.DUPLICATE_ROLE_BYTES, 4 * MIB) + workflow = _text_chunk(b"tEXt", "workflow", b"{}" + b" " * 1000) + size = len(workflow) - 12 + path = _png_with_chunks( + self.directory / "wf.png", [self.prompt_chunk()] + [workflow] * 6 + ) + added = [] + original = prompt_inspector._PngRecords.add + + def counting(records, kind, data): + added.append(len(data)) + return original(records, kind, data) + + with ( + patch.object( + prompt_inspector, "DUPLICATE_WORKFLOW_BYTES", size * 2 + size // 2 + ), + patch.object(prompt_inspector._PngRecords, "add", counting), + ): + read = read_inspection_file(path) + # Original plus two budgeted duplicates were read; the third is limited. + self.assertEqual(added.count(size), 3) + self.assertEqual(read.record_formats["workflow"], "limited") + self.assertNotIn("workflow", read.metadata) + self.assertEqual(read.record_formats["prompt"], "original-text") + self.assert_prompts_resolved(read) + + def test_role_duplicates_past_their_budget_fail_closed(self): + from utils import prompt_inspector + + chunk = self.prompt_chunk() + size = len(chunk) - 12 + path = _png_with_chunks(self.directory / "dupes.png", [chunk] * 6) + with patch.object( + prompt_inspector, "DUPLICATE_ROLE_BYTES", size * 2 + size // 2 + ): + with self.assertRaisesRegex( + MetadataReadError, "Metadata exceeds inspection limits" + ): + read_inspection_file(path) + + def test_over_budget_role_record_never_falls_back_to_other_evidence(self): + from utils import prompt_inspector + + chunk = self.prompt_chunk() + size = len(chunk) - 12 + extras = [ + _text_chunk(b"tEXt", "parameters", b"fallback parameters"), + _text_chunk(b"tEXt", "workflow", b"{}"), + ] + path = _png_with_chunks(self.directory / "fallback.png", [chunk] * 6 + extras) + with patch.object(prompt_inspector, "DUPLICATE_ROLE_BYTES", size * 2): + with self.assertRaisesRegex( + MetadataReadError, "Metadata exceeds inspection limits" + ): + read_inspection_file(path) + + def test_role_key_can_never_end_up_limited(self): + from utils import prompt_inspector + + path = _png_with_chunks(self.directory / "plain.png", [self.prompt_chunk()]) + original = prompt_inspector._PngRecords.skip + + def limit_role(records, key, length): + records.limited.add("prompt") + return original(records, key, length) + + with patch.object(prompt_inspector._PngRecords, "skip", limit_role): + with self.assertRaisesRegex( + MetadataReadError, "Metadata exceeds inspection limits" + ): + read_inspection_file(path) + + def test_duplicates_within_the_total_budget_are_unchanged(self): + path = _png_with_chunks(self.directory / "few.png", [self.prompt_chunk()] * 3) + read = read_inspection_file(path) + self.assertEqual(read.record_formats["prompt"], "original-text") + self.assert_prompts_resolved(read) + + def test_unretained_text_chunks_are_skipped_without_retention(self): + software = _text_chunk(b"tEXt", "Software", b"S" * (5 * MIB)) + compressed = _text_chunk(b"zTXt", "Comment", b"\0" + b"not zlib at all") + path = _png_with_chunks( + self.directory / "software.png", + [software, compressed, self.prompt_chunk()], + ) + reads = self.tracked_reads(path) + self.assertLess(sum(reads), 64 * 1024) + read = read_inspection_file(path) + self.assertEqual(set(read.metadata), {"prompt"}) + + def test_unretained_chunk_with_malformed_keyword_still_fails(self): + for body in (b"no-separator", b"k" * 100 + b"\0text", b"\0text"): + with self.subTest(body=body[:12]): + path = _png_with_chunks( + self.directory / "keyword.png", + [self.prompt_chunk(), parity._raw_chunk(b"tEXt", body)], + ) + with self.assertRaisesRegex(MetadataReadError, "Malformed"): + read_inspection_file(path) + + def test_chunk_count_limit_unchanged(self): + from utils import prompt_inspector + + self.assertEqual(prompt_inspector.MAX_PNG_CHUNKS, 10_000) + filler = _text_chunk(b"tEXt", "Comment", b"x") + for total, readable in ((10_000, True), (10_001, False)): + with self.subTest(total=total): + path = _png_with_chunks(self.directory / "count.png", []) + structural = self.chunk_count(path) + path = _png_with_chunks( + self.directory / "count.png", + [self.prompt_chunk()] + [filler] * (total - structural - 1), + ) + self.assertEqual(self.chunk_count(path), total) + if readable: + self.assert_prompts_resolved(read_inspection_file(path)) + else: + with self.assertRaisesRegex(MetadataReadError, "chunk count"): + read_inspection_file(path) + + @staticmethod + def chunk_count(path): + data, offset, count = path.read_bytes(), 8, 0 + while offset < len(data): + (length,) = struct.unpack(">I", data[offset : offset + 4]) + offset += 12 + length + count += 1 + return count + + def test_non_png_workflow_over_budget_is_limited_not_unreadable(self): + from utils.prompt_inspector import WORKFLOW_METADATA_BYTES + + prompt = json.dumps(parity.build("f04")["prompt"]) + metadata = {"prompt": prompt, "workflow": "w" * (WORKFLOW_METADATA_BYTES + 1)} + fake = MagicMock() + fake.__enter__.return_value = fake + fake.info, fake.format, fake.size = metadata, "WEBP", (4, 4) + path = self.directory / "image.webp" + path.write_bytes(b"RIFF0000WEBP") + with patch("utils.prompt_inspector.Image.open", return_value=fake): + read = read_inspection_file(path) + self.assertEqual(read.limited_records, frozenset({"workflow"})) + self.assertEqual( + read.record_formats, {"prompt": "original-text", "workflow": "limited"} + ) + self.assert_prompts_resolved(read) + + def test_limited_record_format_omits_raw_and_limits_graph(self): + read = InspectionRead( + metadata={"prompt": graph_text(), "workflow": "{}"}, + record_formats={"prompt": "original-text", "workflow": "original-text"}, + video_carrier_text=None, + format="PNG", + width=4, + height=4, + limited_records=frozenset({"workflow"}), + ) + metadata = json.loads(encode(read))["metadata"] + self.assertEqual(metadata["record_formats"]["workflow"], "limited") + self.assertNotIn("workflow", metadata["raw"]) + self.assertEqual( + metadata["workflow_graph"], {"status": "limited", "value": None} + ) + self.assertEqual(metadata["prompt_graph"]["status"], "parsed") + only = InspectionRead({}, {"workflow": "limited"}, None, "PNG", 4, 4) + metadata = json.loads(encode(only))["metadata"] + self.assertEqual(metadata["status"], "available") + self.assertEqual(metadata["raw"], {}) + self.assertEqual(metadata["record_formats"], {"workflow": "limited"}) + self.assertEqual(metadata["workflow_graph"]["status"], "limited") + self.assertEqual(metadata["prompt_graph"]["status"], "absent") diff --git a/tests/test_prompt_pair_resolver.py b/tests/test_prompt_pair_resolver.py index bb4c8cd..3ab9943 100644 --- a/tests/test_prompt_pair_resolver.py +++ b/tests/test_prompt_pair_resolver.py @@ -1,9 +1,17 @@ """Conservative prompt-pair resolution contracts, independent of ComfyUI.""" +import random import unittest +from dataclasses import replace +from utils import prompt_pair_resolver as resolver from utils.prompt_pair_resolver import OutputAdapter, resolve_prompt_pair +try: + from tests import inspection_parity as parity +except ImportError: # discovered with ``-s tests`` + import inspection_parity as parity + def graph(positive: str = "portrait", negative: str = "blurry") -> dict: return { @@ -423,5 +431,617 @@ class TestPromptPairResolver(unittest.TestCase): self.assertEqual(result["state"], "unresolved") +def _f03_metadata() -> dict: + return {"prompt": parity.build("f03")["prompt"]} + + +def _vector(name: str) -> dict: + return next(v for v in parity.load_vectors() if v["name"] == name)["expect"] + + +def _diamond(prefix: str = "d", leaves: int = 575, middles: int = 24) -> dict: + """A ~600-node string diamond: 24 fans over 575 leaves, one leaf shared.""" + letters = "abcdefghijklmnopqrstuvwxyz" + nodes = { + f"{prefix}l{index}": {"class_type": "PrimitiveString", "inputs": {"value": "a"}} + for index in range(leaves) + } + for middle in range(middles): + nodes[f"{prefix}m{middle}"] = { + "class_type": "Text Concatenate", + "inputs": { + f"text_{letters[index]}": [ + f"{prefix}l{(middle * middles + index) % leaves}", + 0, + ] + for index in range(middles) + }, + } + nodes[f"{prefix}r"] = { + "class_type": "Text Concatenate", + "inputs": { + f"text_{letters[index]}": [f"{prefix}m{index}", 0] + for index in range(middles) + }, + } + return nodes + + +def _diamond_samplers(count: int) -> dict: + """``count`` samplers, each encoding the same diamond through its own encoder.""" + source = {"neg": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}}} + source.update(_diamond()) + for index in range(count): + source[f"e{index}"] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": ["dr", 0]}, + } + source[f"s{index}"] = { + "class_type": "KSampler", + "inputs": {"positive": [f"e{index}", 0], "negative": ["neg", 0]}, + } + return source + + +def _distinct_diamond_samplers(count: int, chain: int = 60) -> dict: + """``count`` samplers, each with its own 590-node diamond, sharing a latent chain. + + Real work exceeds the shared 20,000-visit total while the graph stays under + the node ceiling: every candidate re-walks the (unmemoized) latent chain. + """ + source = { + "neg": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}}, + "z0": {"class_type": "EmptyLatentImage", "inputs": {}}, + } + for index in range(1, chain + 1): + source[f"z{index}"] = { + "class_type": "Reroute", + "inputs": {"value": [f"z{index - 1}", 0]}, + } + for index in range(count): + prefix = f"d{index}_" + source.update(_diamond(prefix, leaves=565)) + source[f"e{index}"] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": [f"{prefix}r", 0]}, + } + source[f"s{index}"] = { + "class_type": "KSampler", + "inputs": { + "positive": [f"e{index}", 0], + "negative": ["neg", 0], + "latent_image": [f"z{chain}", 0], + }, + } + return source + + +def _depth_straddling_samplers() -> dict: + """A at depth 9 below its sampler, B four Reroutes deeper on the same chain.""" + source = {"p": {"class_type": "PrimitiveString", "inputs": {"value": "cat"}}} + previous = "p" + for index in range(6): + source[f"c{index}"] = { + "class_type": "Reroute", + "inputs": {"value": [previous, 0]}, + } + previous = f"c{index}" + source["encA"] = {"class_type": "CLIPTextEncode", "inputs": {"text": [previous, 0]}} + source["A"] = { + "class_type": "KSampler", + "inputs": {"positive": ["encA", 0], "negative": ["encA", 0]}, + } + for index in range(4): + source[f"b{index}"] = { + "class_type": "Reroute", + "inputs": {"value": [previous, 0]}, + } + previous = f"b{index}" + source["encB"] = {"class_type": "CLIPTextEncode", "inputs": {"text": [previous, 0]}} + source["B"] = { + "class_type": "KSampler", + "inputs": {"positive": ["encB", 0], "negative": ["encB", 0]}, + } + return source + + +def _nested_shared_chain_samplers() -> dict: + """A, B and C share one chain; each later sampler hangs below a memo hit.""" + source = {"p": {"class_type": "PrimitiveString", "inputs": {"value": "cat"}}} + + def chain(start: str, prefix: str, count: int) -> str: + previous = start + for index in range(count): + source[f"{prefix}{index}"] = { + "class_type": "Reroute", + "inputs": {"value": [previous, 0]}, + } + previous = f"{prefix}{index}" + return previous + + def sampler(name: str, text_source: str) -> None: + source[f"enc{name}"] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": [text_source, 0]}, + } + source[name] = { + "class_type": "KSampler", + "inputs": {"positive": [f"enc{name}", 0], "negative": [f"enc{name}", 0]}, + } + + top = chain("p", "c", 6) # A memoizes c5 with height 6 + sampler("A", top) + middle = chain(top, "r", 1) # r0 is built over a memo hit on c5 + sampler("B", middle) + sampler("C", chain(middle, "d", 3)) # C reaches r0 three Reroutes deeper + return source + + +def _random_graph(rng: random.Random) -> dict: + ids = [str(index) for index in range(rng.randrange(5, 60))] + classes = ( + "StringConcatenate", + "Text Concatenate", + "Reroute", + "PrimitiveString", + "CLIPTextEncode", + "KSampler", + "PromptManager", + "EmptyLatentImage", + ) + + def link(): + return [rng.choice(ids), 0] + + source = {} + for node_id in ids: + name = rng.choice(classes) + if name == "StringConcatenate": + inputs = {"string_a": link(), "string_b": rng.choice([link(), "x"])} + elif name == "Text Concatenate": + inputs = {"text_a": link(), "text_b": link(), "delimiter": ""} + elif name == "Reroute": + inputs = {"value": link()} + elif name == "PrimitiveString": + inputs = {"value": rng.choice(["a", " b "])} + elif name == "CLIPTextEncode": + inputs = {"text": rng.choice([link(), "p"])} + elif name == "KSampler": + inputs = {"positive": link(), "negative": link()} + if rng.random() < 0.3: + inputs["latent_image"] = link() + elif name == "PromptManager": + inputs = {"text": rng.choice(["q", link()]), "prepend_text": ""} + else: + inputs = {} + source[node_id] = {"class_type": name, "inputs": inputs} + return source + + +def _outcomes(result: dict) -> dict: + items = result["candidates"] or [result] + return { + item["sampler_id"]: (item["state"], item["reason_code"], item["reason"]) + for item in items + } + + +class _CountingInputs(dict): + """Node inputs that count how often a resolver iterates (scans) their keys.""" + + scans = 0 + + def __iter__(self): + type(self).scans += 1 + return super().__iter__() + + +class _CountingText(str): + """A literal that tallies the characters a resolver strips from it.""" + + stripped = 0 + + def strip(self, *args): + type(self).stripped += len(self) + return super().strip(*args) + + +class TestResolverProfiles(unittest.TestCase): + """Inspection bounds work, not graph size; the default profile is pinned.""" + + def test_inspection_profile_resolves_reachable_branch_in_506_node_graph(self): + expected = _vector("f03-default") + self.assertGreater(len(_f03_metadata()["prompt"]), 500) + result = resolve_prompt_pair( + _f03_metadata(), profile=resolver.INSPECTION_PROFILE + ) + self.assertEqual(result["state"], "resolved") + self.assertIsNone(result["reason_code"]) + self.assertEqual(result["positive"]["text"], parity.POS) + self.assertEqual(result["negative"]["text"], parity.NEG) + self.assertEqual(expected["state"], "unresolved") # ingestion baseline + + def test_default_profile_keeps_500_node_gate(self): + for result in ( + resolve_prompt_pair(_f03_metadata()), + resolve_prompt_pair(_f03_metadata(), profile=resolver.DEFAULT_PROFILE), + ): + self.assertEqual(result["state"], "unresolved") + self.assertEqual(result["reason"], "Resolution limit reached") + self.assertIsNone(result["reason_code"]) + self.assertNotIn("node", result) + + def test_total_visit_budget_is_shared_across_candidates(self): + source = _distinct_diamond_samplers(33) + self.assertEqual(resolver.INSPECTION_PROFILE.total_visits, 20_000) + self.assertLess(len(source), resolver.INSPECTION_PROFILE.node_ceiling) + result = resolve_prompt_pair( + {"prompt": source}, profile=resolver.INSPECTION_PROFILE + ) + # While selection is off (until PR3) many samplers are always ambiguous; + # PR3 must add "work-limit never upgrades to resolved" once one is chosen. + self.assertNotEqual(result["state"], "resolved") + self.assertEqual(result["state"], "ambiguous") + candidates = result["candidates"] + self.assertEqual(len(candidates), 33) + self.assertEqual(candidates[0]["state"], "resolved") + codes = [candidate["reason_code"] for candidate in candidates] + first_limited = codes.index("work-limit") + self.assertGreater(first_limited, 0) + self.assertTrue(all(code == "work-limit" for code in codes[first_limited:])) + self.assertTrue( + all(c["state"] == "unresolved" for c in candidates[first_limited:]) + ) + # The default profile keeps its historical gate on the same graph. + self.assertEqual( + resolve_prompt_pair({"prompt": source})["reason"], + "Resolution limit reached", + ) + + def test_memo_hit_respects_the_depth_left_at_the_reusing_candidate(self): + source = _depth_straddling_samplers() + reverse = dict(reversed(list(source.items()))) + profile = resolver.INSPECTION_PROFILE + forward = _outcomes( + resolve_prompt_pair({"prompt": source}, profile=profile, max_depth=10) + ) + backward = _outcomes( + resolve_prompt_pair({"prompt": reverse}, profile=profile, max_depth=10) + ) + self.assertEqual(forward, backward) + self.assertEqual(forward["A"][:2], ("resolved", None)) + self.assertEqual(forward["B"][:2], ("unresolved", "depth-limit")) + anchored = resolve_prompt_pair( + {"prompt": source}, profile=profile, max_depth=10, output_node_id="B" + ) + self.assertEqual( + (anchored["state"], anchored["reason_code"]), ("unresolved", "depth-limit") + ) + + def test_memo_height_propagates_through_a_nested_memo_hit(self): + source = _nested_shared_chain_samplers() + reverse = dict(reversed(list(source.items()))) + profile = resolver.INSPECTION_PROFILE + for graph in (source, reverse): + outcomes = _outcomes( + resolve_prompt_pair({"prompt": graph}, profile=profile, max_depth=12) + ) + self.assertEqual(outcomes["C"][:2], ("unresolved", "depth-limit")) + anchored = resolve_prompt_pair( + {"prompt": source}, profile=profile, max_depth=12, output_node_id="C" + ) + self.assertEqual( + (anchored["state"], anchored["reason_code"]), ("unresolved", "depth-limit") + ) + + def test_candidate_outcomes_match_anchored_and_ignore_node_order(self): + profile = resolver.INSPECTION_PROFILE + limits = ({}, {"max_depth": 4}, {"max_depth": 8}, {"max_visits": 6}) + compared = 0 + for seed in range(240, 246): # 6,000 deterministic graphs, about 1.5 s + rng = random.Random(seed) + for _ in range(1_000): + source = _random_graph(rng) + kwargs = rng.choice(limits) + base = _outcomes( + resolve_prompt_pair({"prompt": source}, profile=profile, **kwargs) + ) + items = list(source.items()) + rng.shuffle(items) + shuffled = _outcomes( + resolve_prompt_pair( + {"prompt": dict(items)}, profile=profile, **kwargs + ) + ) + self.assertEqual(base, shuffled, (seed, source, kwargs)) + if len(base) < 2: + continue + for sampler_id, outcome in base.items(): + anchored = resolve_prompt_pair( + {"prompt": source}, + profile=profile, + output_node_id=sampler_id, + **kwargs, + ) + self.assertEqual( + _outcomes(anchored)[sampler_id], + outcome, + (seed, sampler_id, source, kwargs), + ) + compared += 1 + self.assertGreater(compared, 1_000) + + def test_provenance_entry_budget_reports_the_path_limit(self): + letters = "abcdefghijklmnopqrstuvwxyz" + source = { + "p": {"class_type": "PrimitiveString", "inputs": {"value": "cat"}}, + "neg": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}}, + } + previous = "p" + for index in range(60): + source[f"c{index}"] = { + "class_type": "Reroute", + "inputs": {"value": [previous, 0]}, + } + previous = f"c{index}" + for index in range(500): + source[f"w{index}"] = { + "class_type": "Text Concatenate", + "inputs": {f"text_{c}": [previous, 0] for c in letters}, + } + source[f"e{index}"] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": [f"w{index}", 0]}, + } + source[f"s{index}"] = { + "class_type": "KSampler", + "inputs": {"positive": [f"e{index}", 0], "negative": ["neg", 0]}, + } + result = resolve_prompt_pair( + {"prompt": source}, profile=resolver.INSPECTION_PROFILE + ) + reasons = [candidate["reason"] for candidate in result["candidates"]] + self.assertEqual(result["candidates"][0]["state"], "resolved") + first = reasons.index("Provenance path limit reached") + self.assertGreater(first, 0) + self.assertLessEqual( + set(reasons[first:]), + {"Provenance path limit reached", "Resolution visit limit reached"}, + ) + self.assertEqual(reasons[first], "Provenance path limit reached") + self.assertEqual( + {c["reason_code"] for c in result["candidates"][first:]}, {"work-limit"} + ) + # A small entry budget pins the same reason on one candidate. + small = replace(resolver.INSPECTION_PROFILE, max_path_entries=10) + result = resolve_prompt_pair({"prompt": source}, profile=small) + self.assertEqual( + (result["candidates"][0]["reason"], result["candidates"][0]["reason_code"]), + ("Provenance path limit reached", "work-limit"), + ) + + def test_shared_node_is_scanned_once_however_many_candidates_use_it(self): + _CountingInputs.scans = 0 + fat = _CountingInputs({"text": "portrait"}) + fat.update({f"extra_{index}": 0 for index in range(20_000)}) + source = { + "fat": {"class_type": "CLIPTextEncode", "inputs": fat}, + "neg": {"class_type": "CLIPTextEncode", "inputs": {"text": "bad"}}, + } + for index in range(1_000): + source[f"s{index}"] = { + "class_type": "KSampler", + "inputs": {"positive": ["fat", 0], "negative": ["neg", 0]}, + } + result = resolve_prompt_pair( + {"prompt": source}, profile=resolver.INSPECTION_PROFILE + ) + states = {candidate["state"] for candidate in result["candidates"]} + self.assertEqual(states, {"resolved"}) + # Work is O(nodes), not O(candidates x node size). + self.assertLessEqual(_CountingInputs.scans, 2) + + def test_stripped_concatenation_text_is_charged_to_a_work_budget(self): + _CountingText.stripped = 0 + letters = "abcdefghijklmnopqrstuvwxyz" + hub = _CountingText(" " * 100_000) + source = graph() + source["hub"] = {"class_type": "PrimitiveString", "inputs": {"value": hub}} + for leaf in range(300): + source[f"l{leaf}"] = { + "class_type": "Text Concatenate", + "inputs": {f"text_{c}": ["hub", 0] for c in letters}, + } + for middle in range(12): + source[f"m{middle}"] = { + "class_type": "Text Concatenate", + "inputs": { + f"text_{letters[i]}": [f"l{middle * 25 + i}", 0] for i in range(25) + }, + } + source["root"] = { + "class_type": "Text Concatenate", + "inputs": {f"text_{letters[i]}": [f"m{i}", 0] for i in range(12)}, + } + source["1"]["inputs"]["text"] = ["root", 0] + result = resolve_prompt_pair(source, profile=resolver.INSPECTION_PROFILE) + self.assertEqual( + (result["state"], result["reason_code"]), ("unresolved", "work-limit") + ) + self.assertLessEqual( + _CountingText.stripped, resolver.INSPECTION_PROFILE.max_scanned_units + ) + + def test_per_candidate_visit_budget_resets_for_each_sampler(self): + source = _diamond_samplers(3) + result = resolve_prompt_pair( + {"prompt": source}, profile=resolver.INSPECTION_PROFILE, max_visits=700 + ) + self.assertEqual( + [candidate["state"] for candidate in result["candidates"]], + ["resolved"] * 3, + ) + result = resolve_prompt_pair( + {"prompt": source}, profile=resolver.INSPECTION_PROFILE, max_visits=300 + ) + self.assertEqual( + [candidate["reason_code"] for candidate in result["candidates"]], + ["work-limit"] * 3, + ) + + def test_node_ceiling_reports_graph_ceiling(self): + source = graph() + for index in range(20_001 - len(source)): + source[f"p{index}"] = {"class_type": "PrimitiveInt", "inputs": {"value": 1}} + self.assertEqual(len(source), 20_001) + result = resolve_prompt_pair( + {"prompt": source}, profile=resolver.INSPECTION_PROFILE + ) + self.assertEqual(result["reason_code"], "graph-ceiling") + self.assertNotEqual(result["state"], "resolved") + del source["p0"] + result = resolve_prompt_pair( + {"prompt": source}, profile=resolver.INSPECTION_PROFILE + ) + self.assertEqual(result["state"], "resolved") + + def test_prompt_graph_without_supported_sampler_is_unresolved_not_absent(self): + metadata = {"prompt": parity.build("f17")["prompt"]} + result = resolve_prompt_pair(metadata, profile=resolver.INSPECTION_PROFILE) + self.assertEqual( + (result["state"], result["reason_code"]), ("unresolved", "no-sampler") + ) + self.assertIn("SamplerCustomAdvanced", result["reason"]) + self.assertIsNone(result["positive"]) + default = resolve_prompt_pair(metadata) + self.assertEqual(default["state"], "absent") + self.assertEqual(default["reason"], "No supported sampler pair found") + self.assertIsNone(default["reason_code"]) + + def test_no_sampler_reason_names_at_most_three_sampler_like_classes(self): + source = { + str(index): { + "class_type": f"FutureSampler{index % 5}", + "inputs": {"latent_image": ["x", 0]}, + } + for index in range(10) + } + result = resolve_prompt_pair( + {"prompt": source}, profile=resolver.INSPECTION_PROFILE + ) + self.assertEqual(result["reason_code"], "no-sampler") + self.assertEqual( + result["reason"], + "No supported sampler pair found " + "(saw FutureSampler0, FutureSampler1, FutureSampler2)", + ) + + def test_absent_only_without_prompt_or_parameters(self): + for value in (None, [], "raw text", {}, {"parameters": None}): + with self.subTest(value=value): + result = resolve_prompt_pair(value, profile=resolver.INSPECTION_PROFILE) + self.assertEqual(result["state"], "absent") + self.assertIsNone(result["positive"]) + note = {"1": {"class_type": "Note", "inputs": {"text": "hello"}}} + for value in ( + {"prompt": note}, + note, + {"prompt": {}}, + {"prompt": []}, + {"prompt": {"1": None}}, + {"parameters": "unstructured arbitrary text"}, + ): + with self.subTest(value=value): + result = resolve_prompt_pair(value, profile=resolver.INSPECTION_PROFILE) + self.assertEqual(result["state"], "unresolved") + self.assertIsNotNone(result["reason_code"]) + self.assertEqual(resolve_prompt_pair({"prompt": note})["state"], "absent") + # A pair-level "absent" names the missing negative; it always has a positive. + source = graph() + del source["3"]["inputs"]["negative"] + result = resolve_prompt_pair(source, profile=resolver.INSPECTION_PROFILE) + self.assertEqual(result["state"], "absent") + self.assertEqual(result["positive"]["text"], "portrait") + + def test_existing_limits_map_to_reason_codes(self): + cases = ( + ({"max_visits": 1}, "work-limit", "Resolution visit limit reached"), + ({"max_depth": 0}, "depth-limit", None), + ({"max_depth": 1}, "depth-limit", "Resolution depth limit reached"), + ({"max_text_length": 3}, "text-limit", None), + ) + for limits, code, reason in cases: + with self.subTest(limits=limits): + default = resolve_prompt_pair(graph(), **limits) + self.assertEqual(default["state"], "unresolved") + self.assertIsNone(default["reason_code"]) + result = resolve_prompt_pair( + graph(), profile=resolver.INSPECTION_PROFILE, **limits + ) + self.assertEqual(result["state"], "unresolved") + self.assertEqual(result["reason_code"], code) + if reason: + self.assertEqual(result["reason"], reason) + result = resolve_prompt_pair( + {"parameters": "x" * 20}, + profile=resolver.INSPECTION_PROFILE, + max_text_length=10, + ) + self.assertEqual(result["reason_code"], "text-limit") + + def test_unsupported_node_names_the_blocking_node(self): + source = graph() + source["1"]["class_type"] = "UnknownEncoder" + result = resolve_prompt_pair(source, profile=resolver.INSPECTION_PROFILE) + self.assertEqual(result["reason_code"], "unsupported-node") + self.assertEqual(result["node"], {"id": "1", "class_type": "UnknownEncoder"}) + default = resolve_prompt_pair(source) + self.assertIsNone(default["reason_code"]) + self.assertNotIn("node", default) + self.assertEqual(default["reason"], result["reason"]) + + def test_default_profile_emits_only_a_null_reason_code_beyond_todays_keys(self): + source = graph() + source["4"] = {"class_type": "KSampler", "inputs": source["3"]["inputs"]} + for value in (graph(), source, {"parameters": "a\nSteps: 1, Sampler: x"}): + result = resolve_prompt_pair(value) + self.assertEqual( + set(result), + { + "state", + "positive", + "negative", + "sampler_id", + "reason", + "candidates", + "reason_code", + }, + ) + self.assertIsNone(result["reason_code"]) + for candidate in result["candidates"]: + self.assertNotIn("node", candidate) + self.assertIsNone(candidate["reason_code"]) + + def test_diamond_provenance_paths_are_bounded_in_inspection(self): + source = graph() + source["8"] = {"class_type": "PrimitiveString", "inputs": {"value": ""}} + for index in range(9, 49): + source[str(index)] = { + "class_type": "StringConcatenate", + "inputs": { + "string_a": [str(index - 1), 0], + "string_b": [str(index - 1), 0], + "delimiter": "", + }, + } + source["1"]["inputs"]["text"] = ["48", 0] + result = resolve_prompt_pair(source, profile=resolver.INSPECTION_PROFILE) + self.assertEqual(result["state"], "unresolved") + self.assertEqual(result["reason_code"], "work-limit") + # A memo hit charges its unfolded cost: the 2^k unfolded visits exhaust + # the 2,000 allowance (per-entry paths never outgrow the allowance). + self.assertEqual(result["reason"], "Resolution visit limit reached") + + if __name__ == "__main__": unittest.main() diff --git a/tools/inspection_corpus.py b/tools/inspection_corpus.py new file mode 100644 index 0000000..c3c7cd9 --- /dev/null +++ b/tools/inspection_corpus.py @@ -0,0 +1,672 @@ +"""Read-only inspection corpus tool for the #240 invariants (H1 to H8). + +Runs the workspace current-file inspection path (``inspect_current_file`` then +``encode_inspection_response``) of a code tree on every servable media file under +``--root``, and hands each exact response to the workspace's JS facts. Example:: + + python3 tools/inspection_corpus.py --root OUTPUT_DIR --out SCRATCH/after.jsonl \\ + --workers 8 --baseline SCRATCH/before.jsonl --summary SCRATCH/after.txt + +``--code`` selects the tree whose Python and ``web/js`` are imported (default: the +tree holding this tool), so a copy of the tool can measure an older commit. Run +it against a base checkout first and pass that output as ``--baseline``. + +Read-only by construction: media is opened only for reading, symlinks are not +followed, no database or config is imported, bytecode is not written, and +``--out``/``--summary`` are refused inside the scanned root or a code tree. The +JSONL rows and the summary carry counts, statuses and SHA-256 digests of resolved +role texts, never file names or prompt text, unless ``--with-paths`` is given. +""" + +import argparse +import hashlib +import importlib +import importlib.util +import json +import math +import multiprocessing +import os +import shutil +import stat +import struct +import subprocess # nosec B404 +import sys +import time +from collections import Counter +from pathlib import Path +from types import SimpleNamespace + +TOOL_TREE = Path(__file__).resolve().parents[1] +FACTS_HARNESS = Path(__file__).resolve().with_name("inspection_corpus_facts.mjs") +SERVICE = Path("py", "api", "inspection_service.py") +LIBRARY = {"status": "none"} +RECORD_KEYS = ("prompt", "parameters", "workflow", "comfyassets_prompt_pairs") +ROLE_RECORDS = frozenset({"prompt", "parameters"}) +FACTS_KNOWN_TARGET = 4 +TOP_CLASSES = 20 +BASELINE_P99_MS = 520 +# H8 compares p99 with twice the baseline p99; below this floor a tiny baseline +# (a few ms on fixtures) would only measure scheduler noise. +H8_FLOOR_MS = 50 +P99_FACTOR = 2 +MAX_MS = 5000 +PNG_SIGNATURE = b"\x89PNG\r\n\x1a\n" +PNG_TEXT_CHUNKS = (b"tEXt", b"zTXt", b"iTXt") +MAX_PNG_CHUNKS = 10_000 +HASH_BLOCK = 1 << 20 +MAX_JSON_DEPTH = 64 +EXIT_OK, EXIT_INVARIANT_FAILED, EXIT_REFUSED = 0, 1, 2 + +_STATE = SimpleNamespace(code=None, node=None) + + +class Refused(Exception): + """An argument would make the run write somewhere it must not.""" + + +def _inside(path: Path, area: Path) -> bool: + child = os.path.normcase(os.path.realpath(path)) + parent = os.path.normcase(os.path.realpath(area)) + try: + return os.path.commonpath([child, parent]) == parent + except ValueError: # different drives on Windows + return False + + +def _check_outputs(args) -> None: + areas = [("--root", args.root), ("the code tree", args.code)] + if (TOOL_TREE / SERVICE).is_file(): # a copied tool may live in scratch + areas.append(("this tool's code tree", TOOL_TREE)) + outputs = [("--out", args.out)] + ( + [("--summary", args.summary)] if args.summary else [] + ) + for label, path in outputs: + if Path(path).is_dir(): + raise Refused(f"{label} is a directory") + for name, area in areas: + if _inside(Path(path), area): + raise Refused(f"refusing to write {label} inside {name}") + targets = [os.path.realpath(path) for _, path in outputs] + if args.baseline: + targets.append(os.path.realpath(args.baseline)) + if len(set(map(os.path.normcase, targets))) != len(targets): + raise Refused("--out, --summary and --baseline must be different files") + + +def _load_code(code_dir: Path) -> SimpleNamespace: + """Import only the inspection modules of ``code_dir``: no routes, no database.""" + location = code_dir / SERVICE + if not location.is_file(): + raise Refused("--code is not a PromptManager code tree") + loaded = sys.modules.get("utils") + if loaded is None: + sys.path.insert(0, str(code_dir)) + elif not any(_inside(Path(p), code_dir) for p in loaded.__path__): + raise Refused("another code tree's utils package is already imported") + spec = importlib.util.spec_from_file_location("_corpus_inspection", location) + service = importlib.util.module_from_spec(spec) + spec.loader.exec_module(service) + response = importlib.import_module("utils.inspection_response") + inspector = importlib.import_module("utils.prompt_inspector") + return SimpleNamespace( + inspect=service.inspect_current_file, + extensions=frozenset(service.SERVABLE_EXTENSIONS), + encode=response.encode_inspection_response, + cap=response.MAX_RESPONSE_BYTES, + read=inspector.read_inspection_file, + read_error=inspector.MetadataReadError, + ) + + +def _start_node(node: str | None, code_dir: Path): + if not node: + return None + # Fixed local Node harness, no shell. + return subprocess.Popen( # nosec B603 + [node, str(FACTS_HARNESS), str(code_dir)], + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.DEVNULL, + ) + + +def _close_node(process) -> None: + for stream in (process.stdin, process.stdout): + try: + stream.close() + except OSError: # the pipe of an exited harness + pass + process.wait() + + +def _stop_node() -> None: + process, _STATE.node = _STATE.node, None + if process is not None: + _close_node(process) + + +def _init_worker(code_dir: str, node: str | None) -> None: + sys.dont_write_bytecode = True # never write into the measured code tree + _STATE.code = _load_code(Path(code_dir)) + _STATE.node = _start_node(node, Path(code_dir)) + + +def _facts(body: bytes) -> dict: + process = _STATE.node + if process is None: + return {"facts": None, "facts_error": "node-unavailable"} + try: + process.stdin.write(body + b"\n") + process.stdin.flush() + line = process.stdout.readline() + except OSError: + line = b"" + if not line: + _STATE.node = None + _close_node(process) + return {"facts": None, "facts_error": "node-exited"} + facts = json.loads(line) + if "error" in facts: + return {"facts": None, "facts_error": facts["error"]} + known = sum(1 for fact in facts.values() if fact["status"] == "known") + return {"facts": facts, "facts_known": known} + + +def _digest(text: object) -> str | None: + if not isinstance(text, str): + return None + return hashlib.sha256(text.encode("utf-8", "surrogatepass")).hexdigest() + + +def _roles(prompts: dict) -> dict: + return { + role: { + "status": side.get("status"), + "source": side.get("source"), + "sha256": ( + _digest(side.get("text")) if side.get("status") == "resolved" else None + ), + } + for role, side in prompts.items() + } + + +def _texts(side_pair: dict) -> tuple: + return tuple( + (side_pair.get(role) or {}).get("text") for role in ("positive", "negative") + ) + + +def _resolution(response: dict) -> dict: + # ``candidates`` (and so H2) only appears with the selection slice (PR3); + # PR1 responses carry none, so H2 is vacuous until then. + resolution = response["resolution"] + candidates = resolution.get("candidates") or [] + node = resolution.get("node") + resolved = _texts(response["prompts"]) + return { + "state": resolution.get("state"), + "reason": resolution.get("reason"), + "reason_code": resolution.get("reason_code"), + "selection": resolution.get("selection"), + "node_class": node.get("class_type") if isinstance(node, dict) else None, + "candidate_count": resolution.get("candidate_count", len(candidates)), + "h2": resolution.get("state") == "resolved" + and any(_texts(candidate) != resolved for candidate in candidates), + } + + +def _nonfinite(value: object) -> bool: + stack = [(value, 1)] + while stack: + item, depth = stack.pop() + if isinstance(item, float) and not math.isfinite(item): + return True + if depth < MAX_JSON_DEPTH and isinstance(item, (dict, list)): + children = item.values() if isinstance(item, dict) else item + stack.extend((child, depth + 1) for child in children) + return False + + +def _unique(pairs: list) -> dict: + keys = [key for key, _ in pairs] + if len(set(keys)) != len(keys): + raise ValueError("duplicate key") + return dict(pairs) + + +def malformed_cause(text: str) -> str: + """Why a prompt record may be malformed: ``non-input`` means only quirks outside + node ``inputs`` (``_meta``, ``is_changed``, nonfinite numbers) are left.""" + try: + tree = json.loads(text, object_pairs_hook=_unique) + except (ValueError, RecursionError): + return "unparseable" + nodes = tree.values() if isinstance(tree, dict) else None + if nodes is None or not all( + isinstance(node, dict) + and isinstance(node.get("class_type"), str) + and isinstance(node.get("inputs", {}), dict) + for node in nodes + ): + return "shape" + if any(_nonfinite(node.get("inputs")) for node in nodes): + return "input-nonfinite" + return "non-input" + + +def _hash_chunk(handle, length: int) -> tuple: + digest, keyword, remaining = hashlib.sha256(), b"", length + while remaining: + block = handle.read(min(HASH_BLOCK, remaining)) + if not block: + break + if remaining == length: + keyword = block.split(b"\0", 1)[0] + digest.update(block) + remaining -= len(block) + return keyword.decode("latin-1"), digest.digest() + + +def png_duplicates_identical(path: Path) -> bool | None: + """True when every duplicated record key repeats byte-identical chunk data.""" + seen: dict = {} + with open(path, "rb") as handle: + if handle.read(8) != PNG_SIGNATURE: + return None + for _ in range(MAX_PNG_CHUNKS): + head = handle.read(8) + if len(head) < 8: + break + length, kind = struct.unpack(">I4s", head) + if kind in PNG_TEXT_CHUNKS: + keyword, digest = _hash_chunk(handle, length) + if keyword in RECORD_KEYS: + seen.setdefault(keyword, []).append(digest) + handle.seek(4, os.SEEK_CUR) + else: + handle.seek(length + 4, os.SEEK_CUR) + if kind == b"IEND": + break + repeated = [digests for digests in seen.values() if len(digests) > 1] + if not repeated: + return None + return all(len(set(digests)) == 1 for digests in repeated) + + +def _read_evidence(path: Path, row: dict) -> None: + code = _STATE.code + try: + code.read(path) + row["read_error"] = None + except code.read_error as error: + row["read_error"] = str(error) + if row["read_error"] == "Duplicate prompt metadata" and row["ext"] == ".png": + row["duplicates_identical"] = png_duplicates_identical(path) + + +def _metadata(response: dict, inspection) -> dict: + metadata = response["metadata"] + records = set() if inspection is None else set(inspection.metadata) + records |= set(getattr(inspection, "limited_records", ()) or ()) + row = { + "records": sorted(records & set(RECORD_KEYS)), + "metadata_status": metadata["status"], + "omitted_reason": metadata["omitted_reason"], + "graphs": { + "prompt": metadata["prompt_graph"]["status"], + "workflow": metadata["workflow_graph"]["status"], + }, + "record_formats": metadata["record_formats"], + } + text = None if inspection is None else inspection.metadata.get("prompt") + if row["graphs"]["prompt"] == "malformed" and isinstance(text, str): + row["malformed_cause"] = malformed_cause(text) + return row + + +def _observe(path: Path, rel: str, row: dict) -> None: + code = _STATE.code + row["size"] = path.stat().st_size + _read_evidence(path, row) + started = time.perf_counter() + observed = code.inspect(path, source_label="corpus", saved=None, library=LIBRARY) + image = {"root_index": 0, "path": rel, "filename": path.name} + try: + body = code.encode(image=image, root_scope="0" * 64, **observed) + except ValueError: + row["ms"] = round((time.perf_counter() - started) * 1000, 1) + row["over_cap"] = True + return + row["ms"] = round((time.perf_counter() - started) * 1000, 1) + row["response_bytes"] = len(body) + row["over_cap"] = len(body) > code.cap + response = json.loads(body) + row["prompts"] = _roles(response["prompts"]) + row.update(_resolution(response)) + row.update(_metadata(response, observed["inspection"])) + row.update(_facts(body)) + + +def file_id(rel: str) -> str: + """Stable, name-free join key for ``--baseline`` comparisons.""" + return hashlib.sha256(rel.encode("utf-8", "surrogatepass")).hexdigest()[:32] + + +def inspect_one(task: tuple) -> dict: + path, rel, with_paths = Path(task[0]), task[1], task[2] + row = {"id": file_id(rel), "ext": path.suffix.lower()} + if with_paths: + row["path"] = rel + try: + _observe(path, rel, row) + except Exception as error: # one file never stops the corpus; class only + row["error"] = type(error).__name__ + return row + + +def media_files(root: Path, extensions: frozenset): + """Regular servable files under ``root``; symlinks are neither followed nor read.""" + for directory, subdirectories, names in os.walk(root, followlinks=False): + subdirectories.sort() + for name in sorted(names): + path = Path(directory, name) + if path.suffix.lower() not in extensions: + continue + try: + mode = path.lstat().st_mode + except OSError: + continue + if stat.S_ISREG(mode): + yield str(path), path.relative_to(root).as_posix() + + +def _rows(tasks: list, workers: int, code_dir: Path, node: str | None): + if workers <= 1: + _STATE.node = _start_node(node, code_dir) + try: + for task in tasks: + yield inspect_one(task) + finally: + _stop_node() + return + context = multiprocessing.get_context("spawn") + with context.Pool(workers, _init_worker, (str(code_dir), node)) as pool: + yield from pool.imap_unordered(inspect_one, tasks, chunksize=4) + + +def percentile(values: list, fraction: float) -> float | None: + """Nearest-rank percentile; ``None`` for no values.""" + if not values: + return None + ordered = sorted(values) + return ordered[max(0, math.ceil(fraction * len(ordered)) - 1)] + + +def _fact_reasons(row: dict) -> set: + return {fact.get("reason") for fact in (row.get("facts") or {}).values()} + + +def _h1(rows: list, baseline: list | None) -> dict: + if baseline is None: + return {"status": "N/A", "detail": "no --baseline"} + current = {row["id"]: row for row in rows} + before = [row for row in baseline if row.get("state") == "resolved"] + regressions = Counter() + for old in before: + new = current.get(old["id"]) + if new is None: + regressions["missing"] += 1 + elif new.get("state") != "resolved": + regressions[f"{new.get('state')}:{new.get('reason_code')}"] += 1 + elif _role_digests(new) != _role_digests(old): + regressions["resolved:texts-differ"] += 1 + resolved_now = sum(1 for row in rows if row.get("state") == "resolved") + return { + "status": "FAIL" if regressions else "PASS", + "count": sum(regressions.values()), + "detail": f"resolved at baseline {len(before)}, now {resolved_now}", + "breakdown": dict(sorted(regressions.items())), + } + + +def _role_digests(row: dict) -> tuple: + prompts = row.get("prompts") or {} + return tuple( + ( + (prompts.get(role) or {}).get("status"), + (prompts.get(role) or {}).get("sha256"), + ) + for role in ("positive", "negative") + ) + + +def _false_absent(row: dict) -> bool: + if not ROLE_RECORDS & set(row.get("records") or ()): + return False + positive = ((row.get("prompts") or {}).get("positive") or {}).get("status") + # Negative-only absence beside a resolved positive is honest (ledger ruling). + return ( + positive == "absent" or row.get("state") == "absent" and positive != "resolved" + ) + + +def _counted(count: int, detail: str = "") -> dict: + return {"status": "FAIL" if count else "PASS", "count": count, "detail": detail} + + +def _h8(rows: list, baseline: list | None, cap: int) -> dict: + timings = [row["ms"] for row in rows if "ms" in row] + p99, peak = percentile(timings, 0.99), max(timings, default=None) + base = percentile([row["ms"] for row in baseline or () if "ms" in row], 0.99) + limit = max( + P99_FACTOR * (base if base is not None else BASELINE_P99_MS), H8_FLOOR_MS + ) + over_cap = sum(1 for row in rows if row.get("over_cap")) + failed = (p99 or 0) > limit or (peak or 0) >= MAX_MS or over_cap + return { + "status": "FAIL" if failed else "PASS", + "count": over_cap, + "detail": f"p99 {p99} ms (limit {limit}), max {peak} ms (limit <{MAX_MS}), " + f"responses over the {cap}-byte cap {over_cap}", + } + + +def _h3(rows: list) -> dict: + limited = sum( + 1 + for row in rows + if row.get("read_error") is None and "projection-limited" in _fact_reasons(row) + ) + # Facts that were never computed cannot pass: count them as failures. + missing = sum(1 for row in rows if row.get("facts_error") or row.get("error")) + detail = f"projection-limited {limited}, facts not computed {missing}" + return _counted(limited + missing, detail) + + +def _harness(rows: list) -> dict: + """The tool's own failures, kept apart from inspector outcomes: an unreadable + or unsupported file is a row with ``read_error``/``state``, not an ``error``.""" + crashed = sum(1 for row in rows if row.get("error")) + no_facts = sum(1 for row in rows if row.get("facts_error")) + detail = f"inspection exceptions {crashed}, facts not computed {no_facts}" + return _counted(crashed + no_facts, detail) + + +def invariants(rows: list, baseline: list | None, cap: int) -> dict: + """H1 to H8 from the design; H7 is informational until the video slice.""" + videos = [row for row in rows if row["ext"] == ".mp4"] + with_records = sum(1 for row in videos if row.get("records")) + return { + "H1": _h1(rows, baseline), + "H2": _counted(sum(1 for row in rows if row.get("h2"))), + "H3": _h3(rows), + "H4": _counted( + sum(1 for row in rows if row.get("malformed_cause") == "non-input") + ), + "H5": _counted(sum(1 for row in rows if _false_absent(row))), + "H6": _counted( + sum(1 for row in rows if row.get("duplicates_identical") is True) + ), + "H7": { + "status": "N/A", + "detail": "informational until PR4: mp4 with records " + f"{with_records} of {len(videos)}", + }, + "H8": _h8(rows, baseline, cap), + "harness": _harness(rows), + } + + +def build_report(rows: list, baseline: list | None, cap: int) -> dict: + """Aggregate counts only: no names, no text.""" + timings = [row["ms"] for row in rows if "ms" in row] + unsupported = Counter(row["node_class"] for row in rows if row.get("node_class")) + return { + "files": len(rows), + "extensions": Counter(row["ext"] for row in rows), + "harness_errors": Counter(row["error"] for row in rows if row.get("error")), + "states": Counter(row.get("state") for row in rows if not row.get("error")), + "states_by_extension": Counter( + f"{row['ext']} {row.get('state')}" for row in rows if not row.get("error") + ), + "reason_codes": Counter( + str(row.get("reason_code") or "null") + for row in rows + if not row.get("error") + ), + "selection_kinds": Counter( + str(row.get("selection") or "null") for row in rows if not row.get("error") + ), + "metadata_status": Counter( + row.get("metadata_status") for row in rows if "metadata_status" in row + ), + "read_errors": Counter( + row["read_error"] for row in rows if row.get("read_error") + ), + "malformed_causes": Counter( + row["malformed_cause"] for row in rows if "malformed_cause" in row + ), + "facts_known_at_least_4": sum( + 1 for row in rows if (row.get("facts_known") or 0) >= FACTS_KNOWN_TARGET + ), + "facts_errors": Counter( + row["facts_error"] for row in rows if row.get("facts_error") + ), + "top_unsupported_classes": unsupported.most_common(TOP_CLASSES), + "timing_ms": { + "p50": percentile(timings, 0.5), + "p99": percentile(timings, 0.99), + "max": max(timings, default=None), + }, + "invariants": invariants(rows, baseline, cap), + } + + +def _counter_line(label: str, counts) -> str: + items = counts.most_common() if isinstance(counts, Counter) else counts + body = ", ".join(f"{key} {value}" for key, value in items) or "none" + return f"{label}: {body}" + + +def format_report(report: dict) -> str: + timing = report["timing_ms"] + lines = [ + "Inspection corpus summary", + f"files: {report['files']}", + _counter_line("extensions", report["extensions"]), + _counter_line("harness errors", report["harness_errors"]), + _counter_line("states", report["states"]), + _counter_line("states by extension", report["states_by_extension"]), + _counter_line("reason codes", report["reason_codes"]), + _counter_line("selection kinds", report["selection_kinds"]), + _counter_line("metadata status", report["metadata_status"]), + _counter_line("read errors", report["read_errors"]), + _counter_line("malformed prompt graph causes", report["malformed_causes"]), + f"facts at least {FACTS_KNOWN_TARGET} known: " + f"{report['facts_known_at_least_4']}", + _counter_line("facts errors", report["facts_errors"]), + _counter_line( + f"top {TOP_CLASSES} unsupported classes", report["top_unsupported_classes"] + ), + f"timing ms: p50 {timing['p50']}, p99 {timing['p99']}, max {timing['max']}", + "invariants:", + ] + for name, result in report["invariants"].items(): + count = f" {result['count']}" if "count" in result else "" + detail = f" ({result['detail']})" if result.get("detail") else "" + lines.append(f" {name} {result['status']}{count}{detail}") + for key, value in (result.get("breakdown") or {}).items(): + lines.append(f" {key} {value}") + return "\n".join(lines) + "\n" + + +def _by_id(row: dict) -> str: + return row["id"] + + +def _load_baseline(path: str | None) -> list | None: + if path is None: + return None + with open(path, encoding="utf-8") as handle: + return [json.loads(line) for line in handle if line.strip()] + + +def _parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("--root", required=True, help="media folder, scanned read-only") + parser.add_argument("--out", required=True, help="per-file JSONL to write") + parser.add_argument("--workers", type=int, default=1) + parser.add_argument("--baseline", help="JSONL from an earlier run (H1, H8)") + parser.add_argument("--summary", help="also write the summary text here") + parser.add_argument("--code", default=str(TOOL_TREE), help="code tree to measure") + parser.add_argument("--node", default=shutil.which("node"), help="Node.js binary") + parser.add_argument( + "--with-paths", action="store_true", help="add relative paths to the JSONL" + ) + return parser + + +def main(argv: list | None = None) -> int: + args = _parser().parse_args(argv) + previous, sys.dont_write_bytecode = sys.dont_write_bytecode, True + try: # never write bytecode into the measured code tree + return _run(args) + finally: + sys.dont_write_bytecode = previous + + +def _run(args) -> int: + args.root = Path(os.path.realpath(args.root)) + args.code = Path(os.path.realpath(args.code)) + try: + if not args.root.is_dir(): + raise Refused("--root is not a directory") + if args.node and shutil.which(args.node) is None: + raise Refused("--node is not an executable") + _check_outputs(args) + _STATE.code = _load_code(args.code) + baseline = _load_baseline(args.baseline) + except (Refused, OSError, ValueError) as error: + print(f"inspection_corpus: {error}", file=sys.stderr) + return EXIT_REFUSED + tasks = [ + (path, rel, args.with_paths) + for path, rel in media_files(args.root, _STATE.code.extensions) + ] + rows = sorted(_rows(tasks, args.workers, args.code, args.node), key=_by_id) + with open(args.out, "w", encoding="utf-8", newline="\n") as out: + for row in rows: + out.write(json.dumps(row, ensure_ascii=False, sort_keys=True) + "\n") + report = build_report(rows, baseline, _STATE.code.cap) + text = format_report(report) + sys.stdout.write(text) + if args.summary: + with open(args.summary, "w", encoding="utf-8", newline="\n") as handle: + handle.write(text) + failed = any(result["status"] == "FAIL" for result in report["invariants"].values()) + return EXIT_INVARIANT_FAILED if failed else EXIT_OK + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/inspection_corpus_facts.mjs b/tools/inspection_corpus_facts.mjs new file mode 100644 index 0000000..b0e9024 --- /dev/null +++ b/tools/inspection_corpus_facts.mjs @@ -0,0 +1,28 @@ +// Facts for tools/inspection_corpus.py, through the workspace's own path: +// projectInspectionPayload validates the wire, then extractGenerationFacts reads +// the bounded projection. argv[2] is the code tree whose web/js is loaded. +// stdin: one exact inspection response JSON text per line. +// stdout: one line per input, {field: {status, reason}} or {error}. +import { createRequire } from 'node:module'; +import { pathToFileURL } from 'node:url'; +import path from 'node:path'; +import readline from 'node:readline'; + +const web = path.join(path.resolve(process.argv[2]), 'web', 'js'); +const require = createRequire(import.meta.url); +const { extractGenerationFacts } = require(path.join(web, 'comfy-metadata.js')); +const { projectInspectionPayload } = await import(pathToFileURL(path.join(web, 'workspace', 'inspection-state.mjs')).href); +const identity = { kind: 'rooted-file', key: '0:corpus.png', root_index: 0, path: 'corpus.png', root_scope: null }; + +function facts(line) { + try { + const record = projectInspectionPayload(JSON.parse(line), identity, null, extractGenerationFacts); + return Object.fromEntries(Object.entries(record.generation).map(([field, fact]) => [field, { status: fact.status, reason: fact.reason ?? null }])); + } catch (error) { + return { error: error?.name || 'Error' }; + } +} + +for await (const line of readline.createInterface({ input: process.stdin, crlfDelay: Infinity })) { + process.stdout.write(`${JSON.stringify(facts(line))}\n`); +} diff --git a/tools/vendor/frontend-coverage-baseline.json b/tools/vendor/frontend-coverage-baseline.json index 444263d..4997999 100644 --- a/tools/vendor/frontend-coverage-baseline.json +++ b/tools/vendor/frontend-coverage-baseline.json @@ -241,7 +241,7 @@ }, "workspace/library.mjs": { "lines": 95, - "branches": 84, + "branches": 85, "functions": 96, "statements": 95 }, diff --git a/utils/inspection_response.py b/utils/inspection_response.py index a7cc641..b8a4669 100644 --- a/utils/inspection_response.py +++ b/utils/inspection_response.py @@ -3,7 +3,7 @@ import json import math from decimal import Decimal, InvalidOperation -from typing import TYPE_CHECKING +from typing import TYPE_CHECKING, Literal if TYPE_CHECKING: from .prompt_inspector import InspectionRead @@ -14,7 +14,13 @@ _MAX_SAFE_INTEGER = 2**53 - 1 _RECORD_KEYS = frozenset( {"prompt", "workflow", "parameters", "comfyassets_prompt_pairs"} ) -_FORMATS = frozenset({"original-text", "reserialized-json", "invalid"}) +_FORMATS = frozenset({"original-text", "reserialized-json", "invalid", "limited"}) +_PROMPT_NODE_KEYS = frozenset({"class_type", "inputs"}) +MAX_PROJECTION_DEPTH = 64 +# Backstop, not an expected outcome: executable API nodes for a prompt; UI nodes +# including subgraph definitions for a workflow. +GRAPH_NODE_CEILING = 20_000 +ProjectionKind = Literal["prompt", "workflow", "carrier"] class _ProjectionLimit(ValueError): @@ -30,23 +36,27 @@ def _pairs(items: list[tuple]) -> dict: return result -def _nonfinite(value: str) -> None: - raise ValueError("Nonfinite JSON number") +class _Nonfinite: + """A parsed ``NaN``/``Infinity`` literal: never a fact, projected as null.""" -def _normalize( - value: object, depth: int = 1, *, retain: bool = True, carrier: bool = False -) -> object: - """Walk bounded source once; unrelated carrier fields are only validated.""" +_NONFINITE = _Nonfinite() + + +def _nonfinite(value: str) -> _Nonfinite: + return _NONFINITE + + +def _normalize(value: object, depth: int = 1, *, retain: bool = True) -> object: + """Walk bounded source once; unretained values are only validated as JSON.""" if isinstance(value, (dict, list)): - if depth > 64: + if depth > MAX_PROJECTION_DEPTH: raise _ProjectionLimit() if isinstance(value, dict): result = {} if retain else None for key, child in value.items(): - keep = retain and (not carrier or key in ("prompt", "workflow")) - normalized = _normalize(child, depth + 1, retain=keep) - if keep: + normalized = _normalize(child, depth + 1, retain=retain) + if retain: result[key] = normalized else: result = [] if retain else None @@ -55,17 +65,65 @@ def _normalize( if retain: result.append(normalized) return result + if not retain or value is _NONFINITE: + return None if isinstance(value, Decimal): if not value.is_finite(): raise ValueError("Nonfinite JSON number") # str keeps large exponents compact and never rounds through a float. - return str(value) if retain else None + return str(value) if type(value) is int and abs(value) > _MAX_SAFE_INTEGER: - return str(value) if retain else None - return value if retain else None + return str(value) + return value -def _strict_projection(text: str, *, carrier: bool = False) -> object: +def _project_prompt(tree: object, depth: int = 1) -> dict: + """Each API node keeps exactly ``class_type`` and ``inputs``. + + Other node keys (``_meta``, ``is_changed``, future keys) are validated and + dropped, so a nonfinite value there never reaches the projection. + """ + if not isinstance(tree, dict): + raise ValueError("Prompt graph must be an object of nodes") + if depth + 1 > MAX_PROJECTION_DEPTH: # the root and its node objects + raise _ProjectionLimit() + result = {} + for node_id, node in tree.items(): + if not isinstance(node, dict): + result[node_id] = _normalize(node, depth + 1) + continue + projected = {} + for key, child in node.items(): + keep = key in _PROMPT_NODE_KEYS + normalized = _normalize(child, depth + 2, retain=keep) + if keep: + projected[key] = normalized + result[node_id] = projected + return result + + +def _project(tree: object, kind: str, depth: int = 1) -> object: + if kind == "prompt": + return _project_prompt(tree, depth) + if kind == "workflow": + return _normalize(tree, depth) + if not isinstance(tree, dict): + raise ValueError("Malformed carrier") + result = {} + for key, member in tree.items(): + if key not in ("prompt", "workflow"): + _normalize(member, depth + 1, retain=False) + elif isinstance(member, str): + result[key] = member # projected by its own kind when read + elif key == "prompt" and not isinstance(member, dict): + result[key] = _normalize(member, depth + 1) + else: + result[key] = _project(member, key, depth + 1) + return result + + +def _strict_projection(text: str, *, kind: ProjectionKind) -> object: + """Parse exactly (Decimal, no duplicate keys) and project by record kind.""" try: tree = json.loads( text, @@ -73,34 +131,35 @@ def _strict_projection(text: str, *, carrier: bool = False) -> object: object_pairs_hook=_pairs, parse_constant=_nonfinite, ) - if carrier and not isinstance(tree, dict): - raise ValueError("Malformed carrier") - return _normalize(tree, carrier=carrier) + return _project(tree, kind) except RecursionError: raise _ProjectionLimit() from None except InvalidOperation: raise ValueError("JSON decimal exceeds supported range") from None +def _node_count(value: dict, *, workflow: bool) -> int: + if not workflow: + return len(value) + nodes = value.get("nodes") + count = len(nodes) if isinstance(nodes, list) else 0 + definitions = value.get("definitions") + subgraphs = definitions.get("subgraphs") if isinstance(definitions, dict) else None + for subgraph in subgraphs if isinstance(subgraphs, list) else (): + inner = subgraph.get("nodes") if isinstance(subgraph, dict) else None + count += len(inner) if isinstance(inner, list) else 0 + return count + + def _graph(value: object, *, workflow: bool = False) -> dict: if not isinstance(value, dict): return {"status": "malformed", "value": None} - nodes = value.get("nodes") if workflow else value - if isinstance(nodes, (dict, list)) and len(nodes) > 500: + if _node_count(value, workflow=workflow) > GRAPH_NODE_CEILING: return {"status": "limited", "value": None} return {"status": "parsed", "value": value} def _metadata(inspection: "InspectionRead | None") -> dict: - raw = ( - {} - if inspection is None - else { - key: value - for key, value in inspection.metadata.items() - if key in _RECORD_KEYS and isinstance(value, str) - } - ) formats = ( {} if inspection is None @@ -110,6 +169,22 @@ def _metadata(inspection: "InspectionRead | None") -> dict: if key in _RECORD_KEYS and value in _FORMATS } ) + if inspection is not None: + # A record over its read budget has no text: like ``invalid``, raw omits + # it, and its graph is ``limited`` rather than parsed from nothing. + for key in inspection.limited_records & _RECORD_KEYS: + formats[key] = "limited" + raw = ( + {} + if inspection is None + else { + key: value + for key, value in inspection.metadata.items() + if key in _RECORD_KEYS + and isinstance(value, str) + and formats.get(key) != "limited" + } + ) result = { "status": ( "unreadable" @@ -124,7 +199,7 @@ def _metadata(inspection: "InspectionRead | None") -> dict: carrier, failed = None, None if inspection is not None and inspection.video_carrier_text is not None: try: - carrier = _strict_projection(inspection.video_carrier_text, carrier=True) + carrier = _strict_projection(inspection.video_carrier_text, kind="carrier") except _ProjectionLimit: failed = "limited" except (ValueError, TypeError, OverflowError): @@ -132,8 +207,8 @@ def _metadata(inspection: "InspectionRead | None") -> dict: for key, field in (("prompt", "prompt_graph"), ("workflow", "workflow_graph")): graph = {"status": "absent", "value": None} if key in raw or key in formats: - if failed: - graph["status"] = failed + if failed or formats.get(key) == "limited": + graph["status"] = failed or "limited" elif formats.get(key) == "invalid": graph["status"] = "malformed" else: @@ -141,11 +216,11 @@ def _metadata(inspection: "InspectionRead | None") -> dict: if carrier is not None: value = carrier.get(key) if isinstance(value, str): - value = _strict_projection(value) + value = _strict_projection(value, kind=key) elif formats.get(key) == "reserialized-json": raise _ProjectionLimit() else: - value = _strict_projection(raw[key]) + value = _strict_projection(raw[key], kind=key) graph = _graph(value, workflow=key == "workflow") except _ProjectionLimit: graph["status"] = "limited" @@ -259,11 +334,14 @@ def _encode_inspection( "library": _library_projection(library), } body = _bytes(response) - if len(body) > MAX_RESPONSE_BYTES: - for field in ("prompt_graph", "workflow_graph"): - if metadata[field]["status"] != "absent": - metadata[field] = {"status": "limited", "value": None} - body = _bytes(response) + # Omission order keeps prompt evidence longest: the workflow graph first, + # then the prompt graph, then all raw records atomically. + for field in ("workflow_graph", "prompt_graph"): + if len(body) <= MAX_RESPONSE_BYTES: + break + if metadata[field]["status"] != "absent": + metadata[field] = {"status": "limited", "value": None} + body = _bytes(response) if len(body) > MAX_RESPONSE_BYTES: metadata.update( raw={}, diff --git a/utils/prompt_inspector.py b/utils/prompt_inspector.py index e71f447..f38aa9a 100644 --- a/utils/prompt_inspector.py +++ b/utils/prompt_inspector.py @@ -1,26 +1,44 @@ """Read prompt evidence without inference, indexing, or decoding image pixels.""" +import hashlib import io import json import struct import subprocess # nosec B404 import threading import zlib -from dataclasses import dataclass +from dataclasses import dataclass, field as dataclass_field from pathlib import Path from typing import Any from PIL import Image from .pair_execution import EXECUTION_KEY -from .prompt_pair_resolver import resolve_prompt_pair +from .prompt_pair_resolver import ( + INSPECTION_PROFILE, + ResolverProfile, + resolve_prompt_pair, +) from .video_metadata import ffprobe_path, is_video_path -MAX_METADATA_BYTES = 2 * 1024 * 1024 +# Role records (prompt, parameters, the pair execution record) share one +# budget, encoded and decoded: roles cannot be trusted from a partial record, +# so exceeding it fails the whole read. A workflow has its own budget; past it +# only the workflow is ``limited``. Other text chunks are never retained. +ROLE_METADATA_BYTES = 2 * 1024 * 1024 +WORKFLOW_METADATA_BYTES = 4 * 1024 * 1024 +# Total bytes read for byte-identical duplicate chunks of one file, per class +# so workflow repeats can never starve role records: two budgets each, making +# repeated chunks cost bounded I/O instead of file size. +DUPLICATE_WORKFLOW_BYTES = 2 * WORKFLOW_METADATA_BYTES +DUPLICATE_ROLE_BYTES = 2 * ROLE_METADATA_BYTES +MAX_METADATA_BYTES = ROLE_METADATA_BYTES # video stdout bound, unchanged here +MAX_KEYWORD_BYTES = 80 # a 1-79 byte PNG keyword plus its NUL separator MAX_HEADER_BYTES = 8 * 1024 * 1024 MAX_PNG_CHUNKS = 10_000 _ROLE_METADATA_KEYS = frozenset({"prompt", "parameters", EXECUTION_KEY}) _METADATA_KEYS = _ROLE_METADATA_KEYS | {"workflow"} +_TEXT_CHUNKS = (b"tEXt", b"zTXt", b"iTXt") _SIGNATURE = b"\x89PNG\r\n\x1a\n" @@ -38,6 +56,8 @@ class InspectionRead: format: str | None width: int | None height: int | None + # Records present but over their budget: no text, format ``limited``. + limited_records: frozenset[str] = dataclass_field(default_factory=frozenset) @dataclass(frozen=True) @@ -47,24 +67,33 @@ class _VideoInspection: carrier_text: str | None -def _inflate(data: bytes) -> bytes: +def _inflate(data: bytes, limit: int) -> bytes | None: + """Inflate at most ``limit + 1`` bytes; ``None`` when the output is over limit.""" inflater = zlib.decompressobj() - result = inflater.decompress(data, MAX_METADATA_BYTES + 1) - if len(result) > MAX_METADATA_BYTES or not inflater.eof or inflater.unused_data: + result = inflater.decompress(data, limit + 1) + if len(result) > limit: + return None + if not inflater.eof or inflater.unused_data: raise MetadataReadError("Compressed metadata is invalid or exceeds limits") return result -def _png_text(kind: bytes, data: bytes) -> tuple[str, str]: - keyword, separator, body = data.partition(b"\0") +def _keyword(prefix: bytes) -> str: + keyword, separator, _ = prefix.partition(b"\0") if not separator or not 1 <= len(keyword) <= 79: raise MetadataReadError("Malformed PNG text metadata") - key = keyword.decode("latin-1") + return keyword.decode("latin-1") + + +def _png_text(kind: bytes, data: bytes, limit: int) -> tuple[str, str | None]: + """Decode one retained text chunk; ``None`` text when it inflates past limit.""" + _, _, body = data.partition(b"\0") + key = _keyword(data[:MAX_KEYWORD_BYTES]) encoding = "latin-1" if kind == b"zTXt": if not body or body[0] != 0: raise MetadataReadError("Unsupported PNG text compression") - body = _inflate(body[1:]) + body = _inflate(body[1:], limit) elif kind == b"iTXt": if len(body) < 2 or body[0] not in (0, 1): raise MetadataReadError("Malformed international PNG metadata") @@ -74,21 +103,91 @@ def _png_text(kind: bytes, data: bytes) -> tuple[str, str]: raise MetadataReadError("Malformed international PNG metadata") fields[0].decode("ascii") fields[1].decode("utf-8") - body = _inflate(fields[2]) if flag else fields[2] + body = _inflate(fields[2], limit) if flag else fields[2] encoding = "utf-8" + if body is None: + return key, None if b"\0" in body: raise MetadataReadError("Malformed PNG text metadata") return key, body.decode(encoding) -def _png_metadata(stream: Any, size: int) -> tuple[dict, int, int]: - """CRC-check text and structural chunks, seeking past compressed pixels. +class _PngRecords: + """Retained records with per-class budgets and byte-identical duplicates.""" + + def __init__(self) -> None: + self.found: dict[str, str] = {} + self.limited: set[str] = set() + self.digests: dict[str, tuple[int, bytes]] = {} + self.role_encoded = self.role_decoded = 0 + self.duplicate_workflow = self.duplicate_role = 0 + + def skip(self, key: str, length: int) -> bool: + """Decide from the keyword and length alone whether to seek past a chunk.""" + if key not in _METADATA_KEYS or key in self.limited: + return True # never retained, or a duplicate of a limited workflow + if key in self.digests: + if self.digests[key][0] != length: + raise MetadataReadError("Duplicate prompt metadata") + if key == "workflow": + if self.duplicate_workflow + length > DUPLICATE_WORKFLOW_BYTES: + self.limited.add(key) + self.found.pop(key, None) + return True + self.duplicate_workflow += length + else: + if self.duplicate_role + length > DUPLICATE_ROLE_BYTES: + raise MetadataReadError("Metadata exceeds inspection limits") + self.duplicate_role += length + return False # same length: read and compare bytes + if key == "workflow": + if length > WORKFLOW_METADATA_BYTES: + self.limited.add(key) + return True + return False + self.role_encoded += length + if self.role_encoded > ROLE_METADATA_BYTES: + raise MetadataReadError("Metadata exceeds inspection limits") + return False + + def add(self, kind: bytes, data: bytes) -> None: + key = _keyword(data[:MAX_KEYWORD_BYTES]) + digest = hashlib.sha256(kind + data).digest() + if key in self.digests: + if self.digests[key][1] != digest: + raise MetadataReadError("Duplicate prompt metadata") + return + self.digests[key] = (len(data), digest) + workflow = key == "workflow" + limit = ( + WORKFLOW_METADATA_BYTES + if workflow + else ROLE_METADATA_BYTES - self.role_decoded + ) + _, value = _png_text(kind, data, limit) + size = None if value is None else len(value.encode("utf-8")) + if workflow: + if size is None or size > WORKFLOW_METADATA_BYTES: + self.limited.add(key) + return + else: + self.role_decoded += limit + 1 if size is None else size + if self.role_decoded > ROLE_METADATA_BYTES: + raise MetadataReadError("Metadata exceeds inspection limits") + self.found[key] = value + + +def _png_metadata(stream: Any, size: int) -> tuple[dict, frozenset, int, int]: + """CRC-check retained text and structural chunks, seeking past everything else. PNG text chunks may follow IDAT. Pillow's ``.text`` loads pixels to find - them; this bounded reader instead follows the W3C PNG chunk layout. - IDAT checksums/pixel validity are deliberately outside metadata inspection. + them; this bounded reader instead follows the W3C PNG chunk layout. A text + chunk's keyword is read first (at most ``MAX_KEYWORD_BYTES``); chunks that + are not retained, or are over their budget, are skipped by length, so their + bytes are never held in memory or checksummed. IDAT checksums/pixel + validity are deliberately outside metadata inspection. """ - found, compressed_size, expanded_size = {}, 0, 0 + records = _PngRecords() saw_header = saw_pixels = False for index in range(MAX_PNG_CHUNKS): header = stream.read(8) @@ -99,34 +198,32 @@ def _png_metadata(stream: Any, size: int) -> tuple[dict, int, int]: raise MetadataReadError("Incomplete PNG chunk") if index == 0 and (kind != b"IHDR" or length != 13): raise MetadataReadError("Missing PNG header") - if kind in (b"tEXt", b"zTXt", b"iTXt", b"IHDR", b"IEND"): - compressed_size += length - if compressed_size > MAX_METADATA_BYTES: - raise MetadataReadError("Metadata exceeds inspection limits") - data, checksum = stream.read(length), stream.read(4) - if zlib.crc32(kind + data) != int.from_bytes(checksum, "big"): - raise MetadataReadError("PNG metadata checksum mismatch") - if kind == b"IHDR": - if saw_header or not all(struct.unpack(">II", data[:8])): - raise MetadataReadError("Malformed PNG header") - saw_header = True - width, height = struct.unpack(">II", data[:8]) - elif kind == b"IEND": - if length or not saw_pixels: - raise MetadataReadError("Incomplete PNG image") - return found, width, height - else: - key, value = _png_text(kind, data) - expanded_size += len(value.encode("utf-8")) - if expanded_size > MAX_METADATA_BYTES: - raise MetadataReadError("Metadata exceeds inspection limits") - if key in _METADATA_KEYS: - if key in found: - raise MetadataReadError("Duplicate prompt metadata") - found[key] = value + if kind in _TEXT_CHUNKS: + prefix = stream.read(min(length, MAX_KEYWORD_BYTES)) + if records.skip(_keyword(prefix), length): + stream.seek(length - len(prefix) + 4, 1) + continue + data = prefix + stream.read(length - len(prefix)) + elif kind in (b"IHDR", b"IEND"): + data = stream.read(length) else: saw_pixels |= kind == b"IDAT" stream.seek(length + 4, 1) + continue + checksum = stream.read(4) + if zlib.crc32(kind + data) != int.from_bytes(checksum, "big"): + raise MetadataReadError("PNG metadata checksum mismatch") + if kind == b"IHDR": + if saw_header or not all(struct.unpack(">II", data[:8])): + raise MetadataReadError("Malformed PNG header") + saw_header = True + width, height = struct.unpack(">II", data[:8]) + elif kind == b"IEND": + if length or not saw_pixels: + raise MetadataReadError("Incomplete PNG image") + return records.found, frozenset(records.limited), width, height + else: + records.add(kind, data) raise MetadataReadError("PNG chunk count exceeds inspection limits") @@ -222,7 +319,7 @@ def _video_metadata(path: Path) -> _VideoInspection: def read_inspection_file(path: Path) -> InspectionRead: """Read bounded metadata and header facts without decoding media pixels.""" try: - formats, carrier = {}, None + formats, carrier, limited = {}, None, frozenset() format_name = width = height = None if is_video_path(path): video = _video_metadata(path) @@ -235,7 +332,9 @@ def read_inspection_file(path: Path) -> InspectionRead: with path.open("rb") as stream: signature = stream.read(8) if signature == _SIGNATURE: - metadata, width, height = _png_metadata(stream, path.stat().st_size) + metadata, limited, width, height = _png_metadata( + stream, path.stat().st_size + ) format_name = "PNG" else: stream.seek(0) @@ -243,7 +342,7 @@ def read_inspection_file(path: Path) -> InspectionRead: with Image.open(io.BytesIO(header)) as image: metadata = dict(image.info) format_name, (width, height) = image.format, image.size - result, size = {}, 0 + result, role_size, limited = {}, 0, set(limited) for key in _METADATA_KEYS: if key not in metadata: continue @@ -253,12 +352,31 @@ def read_inspection_file(path: Path) -> InspectionRead: continue if not isinstance(value, str): raise MetadataReadError("Malformed prompt metadata") - size += len(value.encode("utf-8", "surrogatepass")) - if size > MAX_METADATA_BYTES: - raise MetadataReadError("Metadata exceeds inspection limits") + size = len(value.encode("utf-8", "surrogatepass")) + if key == "workflow": + if size > WORKFLOW_METADATA_BYTES: + limited.add(key) + continue + else: + role_size += size + if role_size > ROLE_METADATA_BYTES: + raise MetadataReadError("Metadata exceeds inspection limits") result[key] = value formats.setdefault(key, "original-text") - return InspectionRead(result, formats, carrier, format_name, width, height) + if _ROLE_METADATA_KEYS.intersection(limited): + # A limited role record would vanish and let fallback evidence win. + raise MetadataReadError("Metadata exceeds inspection limits") + for key in limited: + formats[key] = "limited" + return InspectionRead( + result, + formats, + carrier, + format_name, + width, + height, + frozenset(limited), + ) except MetadataReadError: raise except ( @@ -282,8 +400,19 @@ def _side(status: str, text: str | None = None, source: str | None = None) -> di return {"status": status, "text": text, "source": source} -def inspect_prompt_evidence(metadata: dict, saved: dict | None = None) -> dict: - """Keep uncertainty visible; a saved association never replaces file evidence.""" +def inspect_prompt_evidence( + metadata: dict, + saved: dict | None = None, + *, + profile: ResolverProfile = INSPECTION_PROFILE, +) -> dict: + """Keep uncertainty visible; a saved association never replaces file evidence. + + Inspection routes resolve with ``INSPECTION_PROFILE``: bounded work, not + graph size. Ingestion (scan, live monitor, admin import) calls the resolver + directly with its default profile. ``resolution`` carries the closed + ``reason_code`` and, for a blocking node, ``node: {id, class_type}``. + """ metadata = { key: value for key, value in metadata.items() if key in _ROLE_METADATA_KEYS } @@ -291,6 +420,8 @@ def inspect_prompt_evidence(metadata: dict, saved: dict | None = None) -> dict: "state": "unresolved", "sampler_id": None, "reason": "Prompt evidence is malformed", + "reason_code": "malformed", + "node": None, } if not metadata: result = {role: _side("absent") for role in ("positive", "negative")} @@ -311,6 +442,8 @@ def inspect_prompt_evidence(metadata: dict, saved: dict | None = None) -> dict: "state": "absent", "sampler_id": None, "reason": "No parsed prompt metadata", + "reason_code": None, + "node": None, }, } canonical = dict(metadata) @@ -329,7 +462,7 @@ def inspect_prompt_evidence(metadata: dict, saved: dict | None = None) -> dict: }, "resolution": resolution, } - resolved = resolve_prompt_pair(canonical) + resolved = resolve_prompt_pair(canonical, profile=profile) state = resolved["state"] result = {} for role in ("positive", "negative"): @@ -350,6 +483,8 @@ def inspect_prompt_evidence(metadata: dict, saved: dict | None = None) -> dict: "state": state, "sampler_id": resolved.get("sampler_id"), "reason": resolved.get("reason", ""), + "reason_code": resolved.get("reason_code"), + "node": resolved.get("node"), }, } diff --git a/utils/prompt_pair_resolver.py b/utils/prompt_pair_resolver.py index 4db5507..c64a6b8 100644 --- a/utils/prompt_pair_resolver.py +++ b/utils/prompt_pair_resolver.py @@ -14,10 +14,19 @@ Effective text uses a unique bounded execution record for traced paired nodes when available; cached graph-only results remain explicitly inferred. The resolver has no ComfyUI, database, configuration or filesystem dependencies. + +Two ``ResolverProfile`` values exist. ``DEFAULT_PROFILE`` (ingestion: scan, live +monitor, admin import) keeps the historical behavior exactly, including the +500-node gate, and only adds ``reason_code: None``. ``INSPECTION_PROFILE`` bounds +work instead of graph size: a per-candidate visit allowance, a total visit budget +shared by every candidate and attribution walk, a node ceiling backstop, and +bounds on built provenance and text. It also names why a result is not resolved +(``reason_code``, and ``node`` for the first unsupported node), and a prompt graph +without a supported sampler is ``unresolved`` (``no-sampler``), never ``absent``. """ import re -from dataclasses import dataclass +from dataclasses import dataclass, replace from typing import Any, Dict, Mapping, Optional, Tuple from .pair_execution import ExecutionEvidenceError, read_pair_execution @@ -62,8 +71,109 @@ for _name in ("Text Multiline", "String Literal", "ShowText|pysssss"): _DEFAULT_ADAPTERS[(_name, 0)] = OutputAdapter("text", "string") +@dataclass(frozen=True) +class ResolverProfile: + """Resolution bounds. ``legacy_node_gate`` set means the historical code path. + + ``max_path_entries`` and ``max_built_chars`` bound memory across the whole + resolution (provenance entries and newly built text). ``max_scanned_units`` + bounds linear work across the whole resolution: input keys validated or + scanned, characters stripped or joined, and provenance entries copied, each + charged before the work is done. ``None`` disables each bound. + ``semantics`` and ``selection`` are reserved for later slices and stay off. + """ + + name: str + per_candidate_visits: int + total_visits: int + max_depth: int + max_text_length: int + node_ceiling: Optional[int] + legacy_node_gate: Optional[int] + semantics: bool = False + selection: bool = False + max_path_entries: Optional[int] = None + max_built_chars: Optional[int] = None + max_scanned_units: Optional[int] = None + + @property + def legacy(self) -> bool: + return self.legacy_node_gate is not None + + +DEFAULT_PROFILE = ResolverProfile( + name="default", + per_candidate_visits=500, + total_visits=500, + max_depth=64, + max_text_length=100_000, + node_ceiling=None, + legacy_node_gate=500, +) +INSPECTION_PROFILE = ResolverProfile( + name="inspection", + per_candidate_visits=2_000, + total_visits=20_000, + max_depth=64, + max_text_length=100_000, + node_ceiling=20_000, + legacy_node_gate=None, + max_path_entries=64 * 20_000, + max_built_chars=16 * 100_000, + # About four passes over the 2 MiB role-record budget. + max_scanned_units=8_000_000, +) +_SAMPLER_LIKE_INPUTS = ("latent_image", "guider", "positive", "negative") +_NO_SAMPLER_NAMES = 3 +_CLASS_NAME_LENGTH = 64 + + class _Unresolved(Exception): - pass + """A non-resolved outcome; ``code`` is the closed machine vocabulary.""" + + def __init__( + self, message: str, code: Optional[str] = None, node: Optional[dict] = None + ) -> None: + super().__init__(message) + self.code = code + self.node = node + + +def _node_ref(node_id: str, class_type: Any) -> dict: + name = class_type[:_CLASS_NAME_LENGTH] if isinstance(class_type, str) else None + return {"id": node_id[:_CLASS_NAME_LENGTH], "class_type": name} + + +class _Budget: + """Visits shared by every candidate, plus a per-candidate allowance. + + With ``total`` equal to the allowance (the default profile) this is exactly + the historical single counter, so the reset per sampler is a no-op there. + """ + + def __init__(self, total: int) -> None: + self.total = total + self.allowance = total + + def reset(self, allowance: int) -> None: + self.allowance = allowance + + @property + def remaining(self) -> int: + return min(self.total, self.allowance) + + def spend(self) -> None: + # Checked before decrementing, so an exhausted allowance costs no total. + if self.total <= 0 or self.allowance <= 0: + raise _Unresolved("Resolution visit limit reached", "work-limit") + self.total -= 1 + self.allowance -= 1 + + def charge(self, cost: int) -> None: + """Charge a memoized subtree's visit cost to this candidate only.""" + if cost > self.allowance: + raise _Unresolved("Resolution visit limit reached", "work-limit") + self.allowance -= cost def _link(value: Any) -> bool: @@ -85,6 +195,8 @@ def _result( sampler_id: Optional[str] = None, reason: str = "", candidates: Optional[list] = None, + reason_code: Optional[str] = None, + node: Optional[dict] = None, ) -> Dict[str, Any]: return { "state": state, @@ -93,9 +205,19 @@ def _result( "sampler_id": sampler_id, "reason": reason, "candidates": candidates or [], + "reason_code": reason_code, + "node": node, } +def _legacy_result(result: Dict[str, Any]) -> Dict[str, Any]: + """The default profile's wire: today's keys plus ``reason_code: None``.""" + legacy = {key: value for key, value in result.items() if key != "node"} + legacy["reason_code"] = None + legacy["candidates"] = [_legacy_result(item) for item in result["candidates"]] + return legacy + + class _GraphResolver: def __init__( self, @@ -105,10 +227,32 @@ class _GraphResolver: max_depth: int, max_text_length: int, metadata: Optional[dict] = None, + *, + budget: Optional[_Budget] = None, + profile: ResolverProfile = DEFAULT_PROFILE, ) -> None: + # ``max_visits`` is the per-candidate allowance; ``profile`` supplies only + # the path selection and whole-resolution bounds. ``resolve_text_input`` + # relies on the legacy default profile here. self.graph = graph self.adapters = {**_DEFAULT_ADAPTERS, **adapters} - self.budget = max_visits + self.per_candidate_visits = max_visits + self.budget = budget if budget is not None else _Budget(max_visits) + self.budget.reset(max_visits) + self.legacy = profile.legacy + self.path_entries = profile.max_path_entries + self.built_chars = profile.max_built_chars + self.scanned = profile.max_scanned_units + self.checked: dict = {} + # Unfolded visit cost of each memo entry (inspection): a memo hit charges + # it to the current candidate, so each candidate's outcome is independent + # of tracing order while the shared work is done only once. + self.costs: dict = {} + # Height of each memo entry (deepest depth reached below it, relative to + # its own depth): a hit raises ``depth-limit`` exactly where a fresh walk + # from the reusing depth would, so depth too is independent of order. + self.heights: dict = {} + self.deepest = 0 self.max_depth = max_depth self.max_text_length = max_text_length self.memo: dict = {} @@ -117,46 +261,99 @@ class _GraphResolver: self.text_sources: dict = {} def node(self, node_id: str) -> Tuple[str, dict]: + if self.legacy: + return self.checked_node(node_id) + if node_id not in self.checked: + try: + self.checked[node_id] = self.checked_node(node_id) + except _Unresolved as error: + if error.code == "work-limit": + raise + self.checked[node_id] = error + checked = self.checked[node_id] + if isinstance(checked, _Unresolved): + raise _Unresolved(str(checked), checked.code, checked.node) + return checked + + def checked_node(self, node_id: str) -> Tuple[str, dict]: node = self.graph.get(node_id) if not isinstance(node, dict) or not isinstance(node.get("inputs"), dict): - raise _Unresolved("Missing or malformed graph node") + raise _Unresolved("Missing or malformed graph node", "malformed") + self.scan(len(node["inputs"])) if any(not isinstance(key, str) for key in node["inputs"]): - raise _Unresolved("Malformed node input names") + raise _Unresolved("Malformed node input names", "malformed") return str(node.get("class_type", "")), node["inputs"] + def scan(self, units: int) -> None: + """Charge linear work before doing it (inspection; no-op when legacy).""" + if self.scanned is not None: + if units > self.scanned: + self.scanned = -1 + raise _Unresolved("Resolution work limit reached", "work-limit") + self.scanned -= units + + def unsupported(self, message: str, node_id: str) -> _Unresolved: + node = self.graph.get(node_id) + class_type = node.get("class_type") if isinstance(node, dict) else None + if isinstance(class_type, str): + class_type = class_type[:_CLASS_NAME_LENGTH] + return _Unresolved(message, "unsupported-node", _node_ref(node_id, class_type)) + def bounded(self, text: str) -> str: if len(text) > self.max_text_length: - raise _Unresolved("Resolved text exceeds the length limit") + raise _Unresolved("Resolved text exceeds the length limit", "text-limit") + return text + + def built(self, text: str) -> str: + """Charge newly built text against the profile's whole-resolution bound.""" + if self.built_chars is not None: + self.built_chars -= len(text) + if self.built_chars < 0: + raise _Unresolved("Resolution work limit reached", "work-limit") return text def text(self, value: Any, kind: str, depth: int = 0) -> Tuple[str, list, str]: + if not self.legacy: + self.deepest = max(self.deepest, depth) if depth >= self.max_depth: - raise _Unresolved("Resolution depth limit reached") + raise _Unresolved("Resolution depth limit reached", "depth-limit") if isinstance(value, str) and kind == "string": return self.bounded(value), [], value if not _link(value): - raise _Unresolved("Expected a typed graph link or text input") + raise _Unresolved("Expected a typed graph link or text input", "malformed") node_id, slot = str(value[0]), value[1] key = (node_id, slot, kind) if key in self.active: - raise _Unresolved("Cyclic prompt graph") + raise _Unresolved("Cyclic prompt graph", "malformed") if key in self.memo: + if not self.legacy: + height = self.heights[key] + if depth + height >= self.max_depth: + raise _Unresolved("Resolution depth limit reached", "depth-limit") + self.budget.charge(self.costs[key]) + self.deepest = max(self.deepest, depth + height) return self.memo[key] - self.budget -= 1 - if self.budget < 0: - raise _Unresolved("Resolution visit limit reached") - name, inputs = self.node(node_id) - self.active.add(key) + allowance = self.budget.allowance + outer_deepest = self.deepest + self.deepest = depth try: + self.budget.spend() + name, inputs = self.node(node_id) + self.scan(len(inputs)) + self.active.add(key) if name == "Reroute" and slot == 0: keys = [key for key in ("value", "input", "in") if key in inputs] if len(keys) != 1: - raise _Unresolved("Reroute input is not uniquely identifiable") + raise _Unresolved( + "Reroute input is not uniquely identifiable", "malformed" + ) text, path, storage_text = self.text(inputs[keys[0]], kind, depth + 1) elif (name, slot) in self.adapters: adapter = self.adapters[(name, slot)] if not isinstance(adapter, OutputAdapter) or adapter.kind != kind: - raise _Unresolved("Output slot has an incompatible type") + raise _Unresolved( + "Output slot has an incompatible type", "malformed" + ) evidence = ( self.execution_record(node_id, inputs) if name == "PromptManagerPair" @@ -186,11 +383,20 @@ class _GraphResolver: after, after_path, _ = self.text( inputs.get("append_text", ""), "string", depth + 1 ) - text = self.bounded( - " ".join( - part - for part in (before.strip(), text, after.strip()) - if part + self.scan( + len(before) + + len(after) + + len(before_path) + + len(path) + + len(after_path) + ) + text = self.built( + self.bounded( + " ".join( + part + for part in (before.strip(), text, after.strip()) + if part + ) ) ) path = before_path + path + after_path @@ -212,38 +418,56 @@ class _GraphResolver: str(inputs.get("clean_whitespace", "true")).lower() == "true" ) if not keys or not isinstance(delimiter, str): - raise _Unresolved("Malformed concatenation") + raise _Unresolved("Malformed concatenation", "malformed") parts, path, size = [], [], 0 for input_name in keys: part, followed, _ = self.text( inputs.get(input_name, ""), "string", depth + 1 ) + # Charged before stripping: stripped-away text is work too. + self.scan(len(part) + len(followed)) if clean: part = part.strip() path.extend(followed) if part or name == "StringConcatenate": size += len(part) + (len(delimiter) if parts else 0) if size > self.max_text_length: - raise _Unresolved("Resolved text exceeds the length limit") + raise _Unresolved( + "Resolved text exceeds the length limit", "text-limit" + ) parts.append(part) - text = delimiter.join(parts) + text = self.built(delimiter.join(parts)) storage_text = text else: - raise _Unresolved("Unsupported node semantics or output slot") + raise self.unsupported( + "Unsupported node semantics or output slot", node_id + ) # Memoized paths stay bounded even for diamond-shaped string graphs. + self.scan(len(path) + 1) path = [{"node_id": node_id, "output_slot": slot}] + path - if len(path) > self.max_depth * max(1, len(self.graph)): - raise _Unresolved("Provenance path limit reached") + if self.legacy: + path_limit = self.max_depth * max(1, len(self.graph)) + else: + path_limit = self.max_depth * max(1, self.per_candidate_visits) + if self.path_entries is not None: + self.path_entries -= len(path) + if len(path) > path_limit or ( + self.path_entries is not None and self.path_entries < 0 + ): + raise _Unresolved("Provenance path limit reached", "work-limit") self.memo[key] = (text, path, storage_text) + self.costs[key] = allowance - self.budget.allowance + self.heights[key] = self.deepest - depth return text, path, storage_text finally: + self.deepest = max(outer_deepest, self.deepest) self.active.discard(key) def execution_record(self, node_id: str, inputs: dict) -> Optional[dict]: try: record = read_pair_execution(self.metadata, node_id) except ExecutionEvidenceError as error: - raise _Unresolved(str(error)) from error + raise _Unresolved(str(error), "malformed") from error if record is None: return None for name in ("text", "negative_text", "prepend_text", "append_text"): @@ -273,7 +497,8 @@ class _GraphResolver: expected = expected.strip() if expected != record[name]: raise _Unresolved( - "Execution evidence does not match recorded node inputs" + "Execution evidence does not match recorded node inputs", + "malformed", ) return record @@ -296,6 +521,14 @@ class _GraphResolver: def pair(self, sampler_id: str) -> dict: positive = negative = None + self.budget.reset(self.per_candidate_visits) + if not self.legacy and self.budget.total <= 0: + return _result( + "unresolved", + sampler_id=sampler_id, + reason="Resolution visit limit reached", + reason_code="work-limit", + ) try: _, inputs = self.node(sampler_id) # A second sampler in the latent ancestry denotes a multistage run; @@ -315,31 +548,50 @@ class _GraphResolver: negative = self.side(inputs["negative"], sampler_id) return _result("resolved", positive, negative, sampler_id) except _Unresolved as error: - return _result("unresolved", positive, negative, sampler_id, str(error)) + return _result( + "unresolved", + positive, + negative, + sampler_id, + str(error), + reason_code=error.code, + node=error.node, + ) def latent_source(self, value: Any, seen: set, depth: int) -> None: if not _link(value) or depth >= self.max_depth: - raise _Unresolved("Unknown latent ancestry") + code = "malformed" if not _link(value) else "depth-limit" + raise _Unresolved("Unknown latent ancestry", code) node_id, slot = str(value[0]), value[1] - if node_id in seen or len(seen) >= max(0, self.budget): - raise _Unresolved("Cyclic or excessive latent ancestry") + if self.legacy: + if node_id in seen or len(seen) >= max(0, self.budget.remaining): + raise _Unresolved("Cyclic or excessive latent ancestry") + elif node_id in seen: + raise _Unresolved("Cyclic or excessive latent ancestry", "malformed") + else: + self.budget.spend() seen.add(node_id) name, inputs = self.node(node_id) if name in _SAMPLERS: + # ``multistage-differs`` is reserved for when stage texts are compared. raise _Unresolved("Multistage sampler ancestry is ambiguous") if name in ("EmptyLatentImage", "EmptySD3LatentImage") and slot == 0: return if name == "Reroute" and slot == 0 and "value" in inputs: self.latent_source(inputs["value"], seen, depth + 1) return - raise _Unresolved("Unsupported latent ancestry") + raise self.unsupported("Unsupported latent ancestry", node_id) def anchor(self, node_id: str) -> str: seen = set() slot = 0 - while len(seen) < self.max_depth and len(seen) < self.budget: + while len(seen) < self.max_depth and ( + not self.legacy or len(seen) < self.budget.remaining + ): if node_id in seen: - raise _Unresolved("Cyclic output ancestry") + raise _Unresolved("Cyclic output ancestry", "malformed") + if not self.legacy: + self.budget.spend() seen.add(node_id) name, inputs = self.node(node_id) if name in _SAMPLERS and slot == 0: @@ -353,19 +605,28 @@ class _GraphResolver: }.get(name) value = inputs.get(key) if key and slot == 0 else None if not _link(value): - raise _Unresolved("Unsupported output ancestry") + raise self.unsupported("Unsupported output ancestry", node_id) node_id, slot = str(value[0]), value[1] - raise _Unresolved("Output ancestry limit reached") + code = "depth-limit" if len(seen) >= self.max_depth else "work-limit" + raise _Unresolved("Output ancestry limit reached", code) def _parameters(value: Any, max_text_length: int) -> dict: if not isinstance(value, str) or not value: return _result("absent", reason="No A1111 parameters") if len(value) > max_text_length: - return _result("unresolved", reason="Parameters exceed the length limit") + return _result( + "unresolved", + reason="Parameters exceed the length limit", + reason_code="text-limit", + ) footers = list(re.finditer(r"(?m)^Steps:\s*\d+\s*,[^\n]*$", value)) if len(footers) != 1 or value[footers[0].end() :].strip(): - return _result("unresolved", reason="A1111 settings footer is not unique") + return _result( + "unresolved", + reason="A1111 settings footer is not unique", + reason_code="malformed", + ) body = value[: footers[0].start()].rstrip("\r\n") delimiter = "\nNegative prompt:" if body.count(delimiter) > 1: @@ -396,49 +657,136 @@ def _parameters(value: Any, max_text_length: int) -> dict: return _result("resolved", positive_side, negative_side) +def _profile_with( + profile: ResolverProfile, + max_visits: Optional[int], + max_depth: Optional[int], + max_text_length: Optional[int], +) -> ResolverProfile: + """Explicit ``max_*`` keywords override the profile they are given with.""" + changes: Dict[str, Any] = {} + if max_visits is not None: + changes["per_candidate_visits"] = max_visits + if profile.legacy: # one historical number: gate, total and allowance + changes.update(total_visits=max_visits, legacy_node_gate=max_visits) + if max_depth is not None: + changes["max_depth"] = max_depth + if max_text_length is not None: + changes["max_text_length"] = max_text_length + return replace(profile, **changes) if changes else profile + + +def _sampler_like_classes(graph: dict) -> list: + """Distinct classes, in graph order, of nodes shaped like samplers/guiders.""" + names: list = [] + for node in graph.values(): + if not isinstance(node, dict) or not isinstance(node.get("inputs"), dict): + continue + name = node.get("class_type") + if not isinstance(name, str) or not name or name in names: + continue + if any(_link(node["inputs"].get(key)) for key in _SAMPLER_LIKE_INPUTS): + names.append(name[:_CLASS_NAME_LENGTH]) + if len(names) == _NO_SAMPLER_NAMES: + break + return names + + def resolve_prompt_pair( metadata: Any, *, output_node_id: Optional[str] = None, adapters: Optional[Mapping[Tuple[str, int], OutputAdapter]] = None, - max_visits: int = 500, - max_depth: int = 64, - max_text_length: int = 100_000, + profile: ResolverProfile = DEFAULT_PROFILE, + max_visits: Optional[int] = None, + max_depth: Optional[int] = None, + max_text_length: Optional[int] = None, ) -> Dict[str, Any]: """Resolve parsed metadata; a present API graph is authoritative over text. Accepts ``{"prompt": api_graph, "parameters": optional_a1111_text}`` or a direct API graph. An optional saver/sampler node ID anchors branch selection. Unanchored multiple samplers are ambiguous, even when their texts are equal. + ``profile`` selects the bounds; explicit ``max_*`` keywords override it. """ + profile = _profile_with(profile, max_visits, max_depth, max_text_length) + result = _resolve(metadata, output_node_id, adapters, profile) + return _legacy_result(result) if profile.legacy else result + + +def _limit_result(profile: ResolverProfile, graph: dict) -> Optional[Dict[str, Any]]: + if profile.legacy: + if ( + len(graph) > profile.legacy_node_gate + or profile.max_depth <= 0 + or profile.max_text_length <= 0 + ): + return _result("unresolved", reason="Resolution limit reached") + return None + if profile.node_ceiling is not None and len(graph) > profile.node_ceiling: + return _result( + "unresolved", + reason="Prompt graph exceeds the inspection node ceiling", + reason_code="graph-ceiling", + ) + if profile.max_depth <= 0: + return _result( + "unresolved", reason="Resolution limit reached", reason_code="depth-limit" + ) + if profile.max_text_length <= 0: + return _result( + "unresolved", reason="Resolution limit reached", reason_code="text-limit" + ) + return None + + +def _resolve( + metadata: Any, + output_node_id: Optional[str], + adapters: Optional[Mapping[Tuple[str, int], OutputAdapter]], + profile: ResolverProfile, +) -> Dict[str, Any]: if not isinstance(metadata, dict) or not metadata: return _result("absent", reason="No parsed prompt metadata") if "prompt" in metadata: graph = metadata["prompt"] elif "parameters" in metadata: - return _parameters(metadata["parameters"], max_text_length) + return _parameters(metadata["parameters"], profile.max_text_length) else: graph = metadata if not isinstance(graph, dict) or not graph: - return _result("unresolved", reason="API graph is missing or malformed") - if len(graph) > max_visits or max_depth <= 0 or max_text_length <= 0: - return _result("unresolved", reason="Resolution limit reached") + return _result( + "unresolved", + reason="API graph is missing or malformed", + reason_code="malformed", + ) + limited = _limit_result(profile, graph) + if limited is not None: + return limited normalized = {str(node_id): node for node_id, node in graph.items()} if len(normalized) != len(graph): - return _result("unresolved", reason="Graph node IDs are not unique") + return _result( + "unresolved", + reason="Graph node IDs are not unique", + reason_code="malformed", + ) resolver = _GraphResolver( normalized, adapters or {}, - max_visits, - max_depth, - max_text_length, + profile.per_candidate_visits, + profile.max_depth, + profile.max_text_length, metadata if "prompt" in metadata else None, + budget=_Budget(profile.total_visits), + profile=profile, ) if output_node_id is not None: try: sampler_ids = [resolver.anchor(str(output_node_id))] except _Unresolved as error: - return _result("unresolved", reason=str(error)) + return _result( + "unresolved", reason=str(error), reason_code=error.code, node=error.node + ) else: sampler_ids = [ node_id @@ -448,16 +796,7 @@ def resolve_prompt_pair( and node["class_type"] in _SAMPLERS ] if not sampler_ids: - has_pair_inputs = any( - isinstance(node, dict) - and isinstance(node.get("inputs"), dict) - and ("positive" in node["inputs"] or "negative" in node["inputs"]) - for node in normalized.values() - ) - return _result( - "unresolved" if has_pair_inputs else "absent", - reason="No supported sampler pair found", - ) + return _no_sampler(normalized, profile) if len(sampler_ids) > 1: return _result( "ambiguous", @@ -467,6 +806,23 @@ def resolve_prompt_pair( return resolver.pair(sampler_ids[0]) +def _no_sampler(graph: dict, profile: ResolverProfile) -> Dict[str, Any]: + reason = "No supported sampler pair found" + if profile.legacy: + has_pair_inputs = any( + isinstance(node, dict) + and isinstance(node.get("inputs"), dict) + and ("positive" in node["inputs"] or "negative" in node["inputs"]) + for node in graph.values() + ) + return _result("unresolved" if has_pair_inputs else "absent", reason=reason) + # A prompt graph is a record: its prompt is unresolved, never "not recorded". + seen = _sampler_like_classes(graph) + if seen: + reason = f"{reason} (saw {', '.join(seen)})" + return _result("unresolved", reason=reason, reason_code="no-sampler") + + def resolve_text_input( graph: Any, node_id: str, diff --git a/web/js/comfy-metadata.js b/web/js/comfy-metadata.js index c0a4cc2..044dc34 100644 --- a/web/js/comfy-metadata.js +++ b/web/js/comfy-metadata.js @@ -469,8 +469,9 @@ function graphFacts(graph, source, resolution) { if (!object(graph)) return factSet("unresolved", source); + // Graph size is not a bound: each lookup is bounded by its MAX_VISITS seen-set, + // and the server's node ceiling arrives as a `limited` projection. const entries = Object.entries(graph); - if (entries.length > MAX_VISITS) return factSet("unresolved", source, "projection-limited"); if (entries.some(([, node]) => !object(node) || !object(node.inputs) || typeof node.class_type !== "string")) return factSet("unresolved", source); const candidates = entries.filter(([, node]) => GENERATION_SAMPLERS.has(node.class_type)); const selected = candidates.find(([id]) => typeof resolution?.sampler_id === "string" && id === resolution.sampler_id); @@ -560,7 +561,6 @@ if (record.status === "limited") return factSet("unresolved", source, "projection-limited"); if (record.status !== "parsed") return factSet("unresolved", source); if (source === "workflow") { - if (record.value?.nodes?.length > MAX_VISITS) return factSet("unresolved", source, "projection-limited"); if (!validFactWorkflow(record.value)) return factSet("unresolved", source); return graphFacts(workflowToGraph(record.value, true), source, null); } diff --git a/web/js/workspace/file-inspector.mjs b/web/js/workspace/file-inspector.mjs index 012ddcb..a6f048f 100644 --- a/web/js/workspace/file-inspector.mjs +++ b/web/js/workspace/file-inspector.mjs @@ -7,11 +7,14 @@ import { createWorkspaceIcon } from './icons.mjs'; const RAW_KEYS = ['prompt', 'workflow', 'parameters', 'comfyassets_prompt_pairs']; const RAW_LIMIT = 'Raw metadata exceeds the response limit. Download the original file to inspect it externally.'; const CONTAINER_WORKFLOW = 'Original workflow JSON text is unavailable for this container record. Download metadata or the original file.'; +const LIMITED_WORKFLOW = 'Workflow exceeds inspection limits. Download the original file to inspect it externally.'; const REPRESENTATIONS = { 'original-text': 'Original embedded text', 'reserialized-json': 'Reserialized container JSON; numeric spelling/precision may differ', invalid: 'Invalid container record; text is unavailable', + limited: 'Record exceeds inspection limits; text is unavailable', }; +const TEXTLESS = new Set(['invalid', 'limited']); const FACTS = [['seed', 'Seed'], ['steps', 'Steps'], ['cfg', 'CFG'], ['sampler', 'Sampler'], ['scheduler', 'Scheduler'], ['model', 'Model']]; const FACT_SOURCES = {'api-graph': 'Embedded API configuration', workflow: 'Saved workflow configuration', 'embedded-parameters': 'Embedded generation parameters'}; @@ -33,7 +36,7 @@ export function inspectionDownloads(record) { if (Object.hasOwn(REPRESENTATIONS, metadata.record_formats?.[key])) { record_formats[key] = metadata.record_formats[key]; representations[key] = REPRESENTATIONS[record_formats[key]]; - if (record_formats[key] !== 'invalid' && typeof metadata.raw?.[key] === 'string') raw[key] = metadata.raw[key]; + if (!TEXTLESS.has(record_formats[key]) && typeof metadata.raw?.[key] === 'string') raw[key] = metadata.raw[key]; } } const envelope = {inspection_version:1, identity:record.identity, file:record.file, prompts:record.prompts, @@ -42,6 +45,7 @@ export function inspectionDownloads(record) { scope:'Supported embedded records only; one selected container carrier for video. Other metadata and private carrier text are excluded.'}}; result.metadata = {...result.metadata, available:true, reason:null, bodyText:JSON.stringify(envelope, null, 2)}; if (record_formats.workflow === 'reserialized-json') result.workflow.reason = CONTAINER_WORKFLOW; + else if (record_formats.workflow === 'limited') result.workflow.reason = LIMITED_WORKFLOW; else if (record_formats.workflow === 'original-text' && metadata.workflow_graph?.status === 'parsed' && typeof raw.workflow === 'string') { result.workflow = {...result.workflow, available:true, reason:null, bodyText:raw.workflow}; } else result.workflow.reason = 'A valid original workflow JSON record is not available in this file.'; diff --git a/web/js/workspace/inspection-state.mjs b/web/js/workspace/inspection-state.mjs index 628aaad..3997a1d 100644 --- a/web/js/workspace/inspection-state.mjs +++ b/web/js/workspace/inspection-state.mjs @@ -4,6 +4,11 @@ import { promptRoles } from './prompt-blocks.mjs'; const SOURCE_CHANGED = 'The source changed. Refresh the image collection before inspecting this file.'; const RECORD_KEYS = ['prompt', 'workflow', 'parameters', 'comfyassets_prompt_pairs']; const FACT_FIELDS = ['seed', 'steps', 'cfg', 'sampler', 'scheduler', 'model']; +// `invalid` and `limited` records carry no text. Only a workflow can be `limited`: +// role records over their budget fail the whole read on the server. +const RECORD_FORMATS = ['original-text', 'reserialized-json', 'invalid', 'limited']; +const TEXTLESS_FORMATS = ['invalid', 'limited']; +const REASON_CODES = ['work-limit', 'depth-limit', 'text-limit', 'graph-ceiling', 'unsupported-node', 'runtime-value', 'composite', 'no-sampler', 'malformed', 'multistage-differs']; const object = value => value !== null && typeof value === 'object' && !Array.isArray(value); const validScope = value => typeof value === 'string' && /^[a-f0-9]{64}$/.test(value) && value.length === 64; @@ -103,30 +108,43 @@ function metadataFor(value) { (value.parameters !== null && typeof value.parameters !== 'string')) throw new TypeError('Invalid inspection metadata'); const raw = {}, formats = {}; for (const [key, format] of Object.entries(value.record_formats)) { - if (!RECORD_KEYS.includes(key) || !['original-text', 'reserialized-json', 'invalid'].includes(format)) throw new TypeError('Invalid inspection record format'); + if (!RECORD_KEYS.includes(key) || !RECORD_FORMATS.includes(format) || (format === 'limited' && key !== 'workflow')) throw new TypeError('Invalid inspection record format'); formats[key] = format; } for (const [key, text] of Object.entries(value.raw)) { - if (!RECORD_KEYS.includes(key) || typeof text !== 'string' || !Object.hasOwn(formats, key) || formats[key] === 'invalid') throw new TypeError('Invalid inspection raw record'); + if (!RECORD_KEYS.includes(key) || typeof text !== 'string' || !Object.hasOwn(formats, key) || TEXTLESS_FORMATS.includes(formats[key])) throw new TypeError('Invalid inspection raw record'); raw[key] = text; } if (value.parameters !== (raw.parameters ?? null)) throw new TypeError('Invalid inspection parameters projection'); const prompt = graphFor(value.prompt_graph), workflow = graphFor(value.workflow_graph); const wholeRawOmitted = value.status === 'limited'; if (wholeRawOmitted && (Object.keys(raw).length || value.parameters !== null)) throw new TypeError('Invalid atomic raw omission'); - if (!wholeRawOmitted && Object.entries(formats).some(([key, format]) => format !== 'invalid' && !Object.hasOwn(raw, key))) throw new TypeError('Missing represented raw record'); - const invalidGraphOmitted = [['prompt', prompt], ['workflow', workflow]].some(([key, graph]) => formats[key] === 'invalid' && graph.status === 'limited'); + if (!wholeRawOmitted && Object.entries(formats).some(([key, format]) => !TEXTLESS_FORMATS.includes(format) && !Object.hasOwn(raw, key))) throw new TypeError('Missing represented raw record'); + // The response budget omits the workflow graph, then the prompt graph, then all raw. + // An invalid record's graph is otherwise malformed, so a limited one marks omission. + const promptGraphOmitted = formats.prompt === 'invalid' && prompt.status === 'limited'; for (const [key, graph] of [['prompt', prompt], ['workflow', workflow]]) { const hasRecord = Object.hasOwn(formats, key); if ((formats[key] === 'invalid' && !['malformed', 'limited'].includes(graph.status)) || + (formats[key] === 'limited' && graph.status !== 'limited') || hasRecord !== (graph.status !== 'absent')) throw new TypeError('Invalid graph representation provenance'); - if (hasRecord && (wholeRawOmitted || invalidGraphOmitted) && graph.status !== 'limited') throw new TypeError('Invalid atomic graph omission'); + if (hasRecord && (wholeRawOmitted || (promptGraphOmitted && key === 'workflow')) && graph.status !== 'limited') throw new TypeError('Invalid atomic graph omission'); } if ((value.status === 'limited') !== (value.omitted_reason === 'response-size-limit')) throw new TypeError('Invalid metadata omission status'); if (['absent', 'unreadable'].includes(value.status) && (Object.keys(raw).length || Object.keys(formats).length || prompt.status !== 'absent' || workflow.status !== 'absent')) throw new TypeError('Invalid unavailable metadata'); return { status: value.status, omitted_reason: value.omitted_reason, raw, record_formats: formats, prompt_graph: prompt, workflow_graph: workflow, parameters: value.parameters }; } +function resolutionFor(value) { + if (!object(value) || !['resolved', 'absent', 'ambiguous', 'unresolved'].includes(value.state) || + (value.sampler_id !== null && (typeof value.sampler_id !== 'string' || !value.sampler_id)) || typeof value.reason !== 'string') throw new TypeError('Invalid inspection resolution'); + const code = value.reason_code ?? null, node = value.node ?? null; + // Codes explain a non-resolved outcome; a blocking node is named only with its code. + if ((code !== null && (!REASON_CODES.includes(code) || ['resolved', 'absent'].includes(value.state))) || + (node !== null && (code === null || !object(node) || typeof node.id !== 'string' || (node.class_type !== null && typeof node.class_type !== 'string')))) throw new TypeError('Invalid inspection resolution reason'); + return value; +} + function libraryFor(value) { if (!object(value) || !['linked', 'none', 'ambiguous', 'unavailable', 'limited'].includes(value.status) || value.scope !== 'saved-library' || !Array.isArray(value.tags) || value.tags.some(tag => typeof tag !== 'string') || ![null, 'pair', 'positive'].includes(value.metadata_scope)) throw new TypeError('Invalid inspection library'); @@ -163,9 +181,7 @@ export function projectInspectionPayload(result, identity, download, extractFact generation: unavailableFacts(), historyTarget: null, download, }; if (legacy) return record; - const resolution = result.resolution; - if (!object(resolution) || !['resolved', 'absent', 'ambiguous', 'unresolved'].includes(resolution.state) || - (resolution.sampler_id !== null && (typeof resolution.sampler_id !== 'string' || !resolution.sampler_id)) || typeof resolution.reason !== 'string') throw new TypeError('Invalid inspection resolution'); + const resolution = resolutionFor(result.resolution); record.file = fileFor(result.file); record.metadata = metadataFor(result.metadata); record.library = libraryFor(result.library);