diff --git a/TESTING.md b/TESTING.md index 0999efb1..6b44b5e3 100644 --- a/TESTING.md +++ b/TESTING.md @@ -794,6 +794,22 @@ unattributed stage, when it is a plain first stage: not a detailer, no prompt ov set, and no listed sampler in its own latent ancestry (else it stays unattributed) (`tests/test_resolver_saver_anchor.py`, shapes in `tests/saver_anchor_graphs.py` from the maintainer's `gpu_unlock_bench`, dataset and ANIMA runs; browser parity through the `anchor-*` cases, which carry `__file_name`). +Recorded run history (#386): both inspect routes (Images and a history occurrence) +ask `database.run_attribution.recorded_output` inside the file's stat window. It +answers only for a file that exactly one recorded run reported and that was written +during that run (the #342 rules): the one output node that run reported it from (none +when several did) and, per sampler anchor, the effective texts PromptManager recorded +at execution (a role only inferred from the graph is none). Only when the file alone +leaves the prompt open (`ambiguous` or `unresolved`) the inspector resolves again +anchored on that node (`recorded_saver_id`, walked like a file-name match), and keeps +that result only when it names the positive (resolved, or a guider with no +negative); its `graph-inferred` sides read `recorded-run` ("Recorded run" badge). +Still open at one named sampler whose positive text was recorded (a language +model's output feeding PromptManager), the recorded texts show as `execution`. A +result the file resolves alone never changes; a lookup failure is logged and judges +the file alone; local Inspect and ingestion are unchanged +(`tests/test_recorded_saver_anchor.py`, `tests/test_run_attribution.py`, +`tests/test_api_history_outputs.py`). `tools/inspection_corpus.py --root DIR --out JSONL [--workers N] [--baseline JSONL] [--summary TXT] [--code TREE]` measures a media folder read-only against the #240 diff --git a/database/run_attribution.py b/database/run_attribution.py index b7d4ec58..fb969574 100644 --- a/database/run_attribution.py +++ b/database/run_attribution.py @@ -15,6 +15,10 @@ only when all of this holds: Anything else returns None: the caller leaves the file unlinked. Read-only. +``recorded_output`` applies the first two rules and returns, for the +inspector (#386), the one output node (save node) that run reported the file +from and the texts PromptManager recorded at execution per sampler. + Lookups are batched (``stored_run_prompts``): one parameterized query per chunk of at most ``LOOKUP_CHUNK`` distinct file names, matched in memory. The history schema is left exactly as it is (no extra index), so older builds can @@ -26,6 +30,7 @@ import os import sqlite3 import threading import weakref +from dataclasses import dataclass from typing import Any, Callable, Dict, Iterable, List, Optional try: @@ -45,7 +50,8 @@ LOOKUP_CHUNK = 500 _RUNS_FOR_NAMES = ( "SELECT DISTINCT o.filename, o.subfolder, o.root_key, o.path_style, o.run_id, " - "r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at) " + "r.observed_dequeue_at, COALESCE(r.terminal_event_at, r.observed_callback_at), " + "o.output_node_id " "FROM execution_run_outputs o JOIN execution_runs r ON r.run_id = o.run_id " "WHERE o.folder_type = 'output' AND o.filename IN ({marks})" ) @@ -138,9 +144,10 @@ def stored_run_prompt( def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]: - """``{(filename, subfolder, root_key, style): {run_id: (start, end)}}``. + """``{(filename, subfolder, root_key, style): {run_id: (start, end, nodes)}}``. - One query per ``LOOKUP_CHUNK`` names; None without history tables. + ``nodes`` is the set of output node ids that reported the location in that + run. One query per ``LOOKUP_CHUNK`` names; None without history tables. """ found: Dict[tuple, dict] = {} conn = db.model.get_connection() @@ -151,12 +158,49 @@ def _runs_by_location(db: Any, names: List[str]) -> Optional[Dict[tuple, dict]]: rows = conn.execute(query, chunk).fetchall() # nosec B608 except sqlite3.Error: return None # no history tables (a library without history) - for name, subfolder, root_key, style, run_id, began, ended in rows: + for name, subfolder, root_key, style, run_id, began, ended, node in rows: runs = found.setdefault((name, subfolder, root_key, style), {}) - runs[run_id] = (began, ended) + runs.setdefault(run_id, (began, ended, set()))[2].add(node) return found +def _recorded_runs( + db: Any, + image_paths: Iterable[str], + output_root: Optional[str], + path_style: Optional[str], +) -> Dict[str, tuple]: + """``{path: (run_id, output node ids)}`` for each file history attributes. + + A file is attributed when exactly one recorded run reported its location + and it was last written while that run executed. + """ + paths = list(dict.fromkeys(path for path in image_paths if path)) + if not paths or not output_root: + return {} + style = path_style or ("windows" if os.name == "nt" else "posix") + located = {} + for path in paths: + location = _location(path, output_root, style) + if location is not None: + located[path] = location + if not located: + return {} + names = sorted({filename for _key, _sub, filename in located.values()}) + runs = _runs_by_location(db, names) + if not runs: + return {} + attributed = {} + for path, (root_key, subfolder, filename) in located.items(): + reported = runs.get((filename, subfolder, root_key, style), {}) + if len(reported) != 1: + continue + ((run_id, (began, ended, nodes)),) = reported.items() + if _written_during(path, began, ended): + attributed[path] = (run_id, nodes) + return attributed + + def _named_prompt(db: Any, run_id: str) -> Optional[int]: contexts = db.get_execution_contexts(run_id).get("contexts") or [] named = { @@ -183,35 +227,104 @@ def stored_run_prompts( """ paths = list(dict.fromkeys(path for path in image_paths if path)) result: Dict[str, Optional[int]] = dict.fromkeys(paths) - if not paths or not output_root: - return result - style = path_style or ("windows" if os.name == "nt" else "posix") - located = {} - for path in paths: - location = _location(path, output_root, style) - if location is not None: - located[path] = location - if not located: - return result - names = sorted({filename for _key, _sub, filename in located.values()}) - runs = _runs_by_location(db, names) - if not runs: - return result prompts: Dict[str, Optional[int]] = {} - for path, (root_key, subfolder, filename) in located.items(): - reported = runs.get((filename, subfolder, root_key, style), {}) - if len(reported) != 1: - continue - ((run_id, (began, ended)),) = reported.items() - if not _written_during(path, began, ended): - continue + for path, (run_id, _nodes) in _recorded_runs( + db, paths, output_root, path_style + ).items(): if run_id not in prompts: prompts[run_id] = _named_prompt(db, run_id) result[path] = prompts[run_id] return result +@dataclass(frozen=True) +class RecordedOutput: + """What recorded history knows about the run that wrote one file (#386). + + ``saver_id`` is the one output node the run reported the file from (None + when several did). ``texts`` maps each sampler anchor of the run to the + effective texts PromptManager recorded at execution for its roles + (``{"positive": text or None, "negative": text or None}``); a role whose + text was only inferred from the graph is None. + """ + + saver_id: Optional[str] + texts: Dict[str, Dict[str, Optional[str]]] + + +def _recorded_texts(db: Any, run_id: str) -> Dict[str, Dict[str, Optional[str]]]: + texts = {} + for context in db.get_execution_contexts(run_id).get("contexts") or []: + evidence = context.get("evidence") or {} + roles = {} + for role in ("positive", "negative"): + values = context.get(role) or {} + proof = (evidence.get(role) or {}).get("effective") or {} + value = values.get("effective") + recorded = proof.get("provenance") == "recorded" + roles[role] = value if recorded and isinstance(value, str) else None + texts[context["anchor_id"]] = roles + return texts + + +def recorded_output( + db: Any, + image_path: str, + output_root: Optional[str], + path_style: Optional[str] = None, +) -> Optional[RecordedOutput]: + """The recorded run facts for ``image_path``, else None (#386). + + The file must be attributed to one run (``_recorded_runs``: one run + reported its location and it was written during that run). Read-only; it + never links anything. + """ + if not image_path: + return None + found = _recorded_runs(db, [image_path], output_root, path_style) + if image_path not in found: + return None + run_id, nodes = found[image_path] + saver = next(iter(nodes)) if len(nodes) == 1 else None + return RecordedOutput(saver, _recorded_texts(db, run_id)) + + +def recorded_output_node( + db: Any, + image_path: str, + output_root: Optional[str], + path_style: Optional[str] = None, +) -> Optional[str]: + """The one output node history says wrote ``image_path``, else None.""" + found = recorded_output(db, image_path, output_root, path_style) + return found.saver_id if found is not None else None + + _LOGGER = logging.getLogger("prompt_manager.run_attribution") + + +def recorded_output_lookup(db: Any, *, close: bool = False) -> Callable[[Any], Any]: + """``lookup(path) -> RecordedOutput or None`` for the inspect routes (#386). + + Uses ComfyUI's output folder. Any failure (no history, a locked library) + is logged and names no node, so the file is judged from its own metadata. + ``close`` closes the calling thread's connection afterwards, for routes + that close theirs after reading. + """ + + def lookup(path: Any) -> Optional[RecordedOutput]: + try: + return recorded_output(db, str(path), host_output_directory()) + except Exception: + _LOGGER.warning("Recorded run lookup failed", exc_info=True) + return None + finally: + if close: + db.model.close() + + return lookup + + _SEALED_LISTENERS: List[weakref.WeakMethod] = [] _SEALED_LOCK = threading.Lock() diff --git a/py/api/history.py b/py/api/history.py index f50960fb..e7f3affd 100644 --- a/py/api/history.py +++ b/py/api/history.py @@ -31,6 +31,7 @@ from .inspection_service import CurrentFileChanged, inspect_current_file try: from ...database.execution_history import HistoryEpochChanged from ...database.history_gallery import read_history_gallery + from ...database.run_attribution import recorded_output_lookup from ...database.history_reads import ( HistoryCaptureChanged, OutputPosition, @@ -42,6 +43,7 @@ try: except ImportError: from database.execution_history import HistoryEpochChanged from database.history_gallery import read_history_gallery + from database.run_attribution import recorded_output_lookup from database.history_reads import ( HistoryCaptureChanged, OutputPosition, @@ -728,6 +730,7 @@ class HistoryRoutesMixin: "library", {"status": "unavailable" if saved_unavailable else "none"}, ), + recorded=recorded_output_lookup(db, close=True), ) _reauthorize_current_target(resolved, scope) identity = { diff --git a/py/api/images.py b/py/api/images.py index 95d5e2ec..e1ba6051 100644 --- a/py/api/images.py +++ b/py/api/images.py @@ -61,6 +61,7 @@ try: from ...database.image_favorites import MAX_KEY_LENGTH from ...database.library import display_title from ...database.models import normalize_image_path + from ...database.run_attribution import recorded_output_lookup from ...utils.card_seed import card_seed, header_params, parsed_positive from ...utils.parallel import map_parallel from ...utils.json_strict import dumps_strict @@ -81,6 +82,7 @@ except ImportError: from database.image_favorites import MAX_KEY_LENGTH from database.library import display_title from database.models import normalize_image_path + from database.run_attribution import recorded_output_lookup from utils.card_seed import card_seed, header_params, parsed_positive from utils.parallel import map_parallel from utils.json_strict import dumps_strict @@ -3541,6 +3543,7 @@ class ImageRoutesMixin: image_tags=self._inspection_image_tags( roots[index], Path(relative).parts ), + recorded=recorded_output_lookup(self.db), ) except CurrentFileChanged: return None diff --git a/py/api/inspection_service.py b/py/api/inspection_service.py index 5c8e3e9c..3a99af86 100644 --- a/py/api/inspection_service.py +++ b/py/api/inspection_service.py @@ -3,7 +3,7 @@ import os import stat from pathlib import Path -from typing import Callable +from typing import Any, Callable try: from ...utils.prompt_inspector import ( @@ -100,18 +100,23 @@ def inspect_current_file( saved: dict | None, library: dict, image_tags: Callable[[int, int], dict] | None = None, + recorded: Callable[[Path], Any] | None = None, ) -> dict: """Current-file evidence read between two equal stats of *path*. *image_tags*, when given, is called once with that stable ``(st_size, st_mtime_ns)`` and its facet is returned as ``image_tags`` (spec amendment S1: tags of a byte record verified at exactly that stat). + *recorded*, when given, is asked within the same stat window for what + recorded run history knows about *path* (``RecordedOutput`` or None: the + save node that wrote it and the texts PromptManager recorded, #386). """ before = _current_file_facts(path) try: inspection = read_inspection_file(path) except MetadataReadError: inspection = None + run = recorded(path) if recorded is not None and inspection is not None else None after = _current_file_facts(path) fields = ("st_dev", "st_ino", "st_mode", "st_size", "st_mtime_ns", "st_ctime_ns") if any(getattr(before, key) != getattr(after, key) for key in fields): @@ -129,9 +134,15 @@ def inspect_current_file( profile = ( PREVIEW_INSPECTION_PROFILE if _in_host_temp(path) else INSPECTION_PROFILE ) - # The file name anchors on the saver that wrote it (#381). + # The file name anchors on the saver that wrote it (#381); recorded run + # history names that saver when the file alone cannot (#386). evidence = inspect_prompt_evidence( - inspection.metadata, role_saved, profile=profile, file_name=path.name + inspection.metadata, + role_saved, + profile=profile, + file_name=path.name, + recorded_saver_id=run.saver_id if run is not None else None, + recorded_texts=run.texts if run is not None else None, ) return { "file": { diff --git a/tests/js/workspace-file-inspector.test.js b/tests/js/workspace-file-inspector.test.js index 6371f59c..8068c618 100644 --- a/tests/js/workspace-file-inspector.test.js +++ b/tests/js/workspace-file-inspector.test.js @@ -185,6 +185,21 @@ test('legacy fill applies to local records', async () => { assert.notEqual(zeroed.prompts.positive.text, ''); }); +test('a branch named by recorded run history shows the Recorded run badge', async () => { + // #386: the server labels sides whose save node recorded history named; Execution recorded stays stronger. + const base = await localRecord(new File([fs.readFileSync(new URL('f09.png', PARITY))], 'f09.png')); + const recorded = {...base, prompts:{positive:{...base.prompts.positive, source:'recorded-run'}, negative:{...base.prompts.negative, source:'recorded-run'}}}; + let view = renderFull(recorded); + try { + assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Recorded run']); + assert.ok(view.texts(n => n.tagName === 'p').includes('Prompt evidence: recorded run')); + } finally { view.done(); } + view = renderFull({...recorded, prompts:{...recorded.prompts, negative:{...recorded.prompts.negative, source:'execution'}}}); + try { + assert.deepEqual(view.texts(n => n.className === 'chip blue'), ['Execution recorded']); + } finally { view.done(); } +}); + // #249 PR C Task C3 (D88): one chip row, the file's current image tags first, then D21's linked tags not already shown. function tagged(imageTags, library = {status:'linked', tags:['Minimal', 'warm_light', 'editorial'], tags_scope:'positive'}) { return {...record(), imageTags, library}; diff --git a/tests/test_api_history_outputs.py b/tests/test_api_history_outputs.py index c250cc27..c43f43f1 100644 --- a/tests/test_api_history_outputs.py +++ b/tests/test_api_history_outputs.py @@ -21,6 +21,8 @@ from tests.test_api_images import ImageAPITestCase, make_png from tests.test_api_prompt_inspector import graph, png from tests.test_execution_context_database import bundle as context_bundle from tests.test_execution_history_database import JOB, SESSION +from tests.saver_anchor_graphs import BRANCH_TEXTS, shared_prefix +from tests.test_recorded_saver_anchor import LLM_TEXT, text_generate_run from utils.execution_contexts import ContextBundle from utils.execution_history_events import make_finish, make_start from utils.execution_outputs import OutputBundle, OutputReference, snapshot_output_roots @@ -1505,3 +1507,88 @@ class HistoryOutputAPITests(ImageAPITestCase): body = await response.json() self.assertEqual(response.status, 200, body) self.assertEqual(body["prompts"]["positive"]["text"], "Readable portrait") + + async def recorded_shared_prefix_run(self): + """#386: the dataset shape, branch 3's saver wrote the file in a run.""" + path = png(self.output_dir / "IMG_GEN_00371_.png", prompt=shared_prefix()) + # make_start/make_finish observe the run at 0 ms: the file was written then. + os.utime(path, ns=(1_000_000, 1_000_000)) + start = self.capture( + [self.ref(node="33", filename=path.name), self.ref("43", 1, "other.png")] + ) + return start + + async def test_both_inspect_routes_name_the_branch_recorded_history_saved(self): + start = await self.recorded_shared_prefix_run() + _, body = await self.outputs(start) + descriptor = next( + item["current_candidate"]["media"] + for item in body["items"] + if item["occurrence"]["output_node_id"] == "33" + ) + response = await self.current(descriptor["inspection_url"]) + recorded = await response.json() + self.assertEqual(response.status, 200, recorded) + response = await self.client.get( + "/prompt_manager/images/inspect/IMG_GEN_00371_.png", params={"root": "0"} + ) + images = await response.json() + self.assertEqual(response.status, 200, images) + for result in (recorded, images): + self.assertEqual(result["resolution"]["state"], "resolved") + self.assertEqual( + result["prompts"]["positive"], + { + "status": "resolved", + "text": BRANCH_TEXTS[2], + "source": "recorded-run", + }, + ) + + async def test_a_file_rewritten_after_its_run_keeps_several_prompts(self): + await self.recorded_shared_prefix_run() + os.utime(self.output_dir / "IMG_GEN_00371_.png") # written now: not that run + response = await self.client.get( + "/prompt_manager/images/inspect/IMG_GEN_00371_.png", params={"root": "0"} + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual(body["resolution"]["state"], "ambiguous") + self.assertEqual(body["prompts"]["positive"]["status"], "ambiguous") + + async def test_a_failing_history_lookup_judges_the_file_alone(self): + await self.recorded_shared_prefix_run() + with ( + patch( + "database.run_attribution.recorded_output", + side_effect=sqlite3.OperationalError("locked"), + ), + self.assertLogs("prompt_manager.run_attribution", "WARNING"), + ): + response = await self.client.get( + "/prompt_manager/images/inspect/IMG_GEN_00371_.png", + params={"root": "0"}, + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual(body["resolution"]["state"], "ambiguous") + + async def test_a_language_model_text_shows_what_promptmanager_recorded(self): + path = png(self.output_dir / "moss_00026_.png", prompt=text_generate_run()) + os.utime(path, ns=(1_000_000, 1_000_000)) + start = self.capture([self.ref(node="9", filename=path.name)]) + self.api.db.write_execution_contexts( + start.run_id, + start.storage_epoch, + context_bundle("", LLM_TEXT, anchor="7"), + ) + response = await self.client.get( + "/prompt_manager/images/inspect/moss_00026_.png", params={"root": "0"} + ) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual( + body["prompts"]["positive"], + {"status": "resolved", "text": LLM_TEXT, "source": "execution"}, + ) + self.assertEqual(body["prompts"]["negative"]["text"], "") diff --git a/tests/test_recorded_saver_anchor.py b/tests/test_recorded_saver_anchor.py new file mode 100644 index 00000000..c606f561 --- /dev/null +++ b/tests/test_recorded_saver_anchor.py @@ -0,0 +1,309 @@ +"""Recorded run history names the saver that wrote a file (#386, #240). + +When a file's graph alone cannot name its prompt (sampler branches that differ +only in their prompt and share one saver prefix), the inspector asks recorded +history which save node wrote the file and walks from that node, as the #381 +file-name anchor does. A result the file alone resolves is never changed; a +recorded saver that names nothing certain leaves the file "several possible +prompts". The resolved sides are labelled as coming from the recorded run. +""" + +import json +import unittest + +from utils import prompt_pair_resolver as resolver +from utils.prompt_inspector import inspect_prompt_evidence + +try: + from tests import saver_anchor_graphs as shapes +except ImportError: # discovered with ``-s tests`` + import saver_anchor_graphs as shapes + + +def _resolve(graph, **options): + return resolver.resolve_prompt_pair( + {"prompt": graph}, profile=resolver.INSPECTION_PROFILE, **options + ) + + +def _positive(result): + side = result["positive"] + return side["text"] if isinstance(side, dict) else None + + +def _evidence(graph, **options): + return inspect_prompt_evidence({"prompt": json.dumps(graph)}, **options) + + +# ``branch_bench``'s savers are node ``N3`` of branch N (10s based). +def _saver(index): + return str(index * 10 + 3) + + +def guider_dataset(): + """The dataset run's shape: SamplerCustomAdvanced branches on BasicGuider + (no negative), every branch saved with one prefix.""" + graph = { + "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "m"}}, + "2": {"class_type": "EmptyLatentImage", "inputs": {"width": 64}}, + "4": {"class_type": "RandomNoise", "inputs": {"noise_seed": 1}}, + "5": {"class_type": "KSamplerSelect", "inputs": {"sampler_name": "euler"}}, + "6": { + "class_type": "BasicScheduler", + "inputs": {"model": ["1", 0], "steps": 8, "scheduler": "simple"}, + }, + } + for index, text in enumerate(shapes.BRANCH_TEXTS, start=1): + base = index * 10 + graph[str(base)] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": text, "clip": ["1", 1]}, + } + graph[str(base + 4)] = { + "class_type": "BasicGuider", + "inputs": {"model": ["1", 0], "conditioning": [str(base), 0]}, + } + graph[str(base + 1)] = { + "class_type": "SamplerCustomAdvanced", + "inputs": { + "noise": ["4", 0], + "guider": [str(base + 4), 0], + "sampler": ["5", 0], + "sigmas": ["6", 0], + "latent_image": ["2", 0], + }, + } + graph[str(base + 2)] = { + "class_type": "VAEDecode", + "inputs": {"samples": [str(base + 1), 0], "vae": ["1", 2]}, + } + graph[str(base + 3)] = { + "class_type": "SaveImage", + "inputs": { + "filename_prefix": "dataset/IMG_GEN", + "images": [str(base + 2), 0], + }, + } + return graph + + +class TestRecordedSaverInResolver(unittest.TestCase): + def test_the_recorded_saver_picks_its_branch_of_a_shared_prefix(self): + for index, text in enumerate(shapes.BRANCH_TEXTS, start=1): + with self.subTest(branch=index): + result = _resolve( + shapes.shared_prefix(), + file_name="IMG_GEN_00371_.png", + recorded_saver_id=_saver(index), + ) + self.assertEqual(result["state"], "resolved") + self.assertEqual(_positive(result), text) + self.assertEqual(result["sampler_id"], str(index * 10 + 1)) + + def test_without_a_recorded_saver_the_shared_prefix_stays_several(self): + result = _resolve(shapes.shared_prefix(), file_name="IMG_GEN_00371_.png") + self.assertEqual(result["state"], "ambiguous") + + def test_a_recorded_saver_the_graph_lacks_anchors_nothing(self): + for node in ("missing", "3", "2"): + with self.subTest(node=node): + result = _resolve(shapes.shared_prefix(), recorded_saver_id=node) + self.assertEqual(result["state"], "ambiguous") + + def test_a_recorded_saver_of_an_unknown_class_anchors(self): + graph = shapes.branch_bench( + prefixes=["dataset/IMG_GEN"] * len(shapes.BRANCH_TEXTS), + saver="VendorSaveImage", + ) + result = _resolve(graph, recorded_saver_id=_saver(2)) + self.assertEqual(_positive(result), shapes.BRANCH_TEXTS[1]) + + def test_the_default_profile_ignores_a_recorded_saver(self): + result = resolver.resolve_prompt_pair( + {"prompt": shapes.shared_prefix()}, recorded_saver_id=_saver(1) + ) + self.assertEqual(result["state"], "ambiguous") + + +class TestRecordedSaverInInspector(unittest.TestCase): + def test_the_inspector_shows_the_recorded_branch_labelled_as_recorded(self): + evidence = _evidence( + shapes.shared_prefix(), + file_name="IMG_GEN_00371_.png", + recorded_saver_id=_saver(3), + ) + self.assertEqual(evidence["resolution"]["state"], "resolved") + positive = evidence["prompts"]["positive"] + self.assertEqual(positive["text"], shapes.BRANCH_TEXTS[2]) + self.assertEqual(positive["source"], "recorded-run") + self.assertEqual(evidence["prompts"]["negative"]["source"], "recorded-run") + + def test_a_guider_branch_without_a_negative_shows_its_positive(self): + self.assertEqual( + _evidence(guider_dataset(), file_name="IMG_GEN_00371_.png")["resolution"][ + "state" + ], + "ambiguous", + ) + evidence = _evidence( + guider_dataset(), file_name="IMG_GEN_00371_.png", recorded_saver_id="23" + ) + self.assertEqual( + evidence["prompts"]["positive"], + { + "status": "resolved", + "text": shapes.BRANCH_TEXTS[1], + "source": "recorded-run", + }, + ) + self.assertEqual(evidence["prompts"]["negative"]["status"], "absent") + + def test_a_file_that_resolves_alone_keeps_its_own_result(self): + # The file name names branch 2; a recorded saver never overrides it. + evidence = _evidence( + shapes.branch_bench(), + file_name="s45_branch_2_00001_.png", + recorded_saver_id=_saver(4), + ) + positive = evidence["prompts"]["positive"] + self.assertEqual(positive["text"], shapes.BRANCH_TEXTS[1]) + self.assertEqual(positive["source"], "graph-inferred") + + def test_a_recorded_saver_that_names_nothing_keeps_several_prompts(self): + evidence = _evidence(shapes.shared_prefix(), recorded_saver_id="missing") + self.assertEqual(evidence["resolution"]["state"], "ambiguous") + self.assertEqual(evidence["prompts"]["positive"]["status"], "ambiguous") + + def test_no_recorded_saver_is_unchanged(self): + self.assertEqual( + _evidence(shapes.shared_prefix(), recorded_saver_id=None), + _evidence(shapes.shared_prefix()), + ) + + +LLM_TEXT = "a red vintage car parked by the sea, golden hour" + + +def text_generate_run(): + """The maintainer's moss run: a language model writes PromptManager's text.""" + return { + "1": {"class_type": "CheckpointLoaderSimple", "inputs": {"ckpt_name": "m"}}, + "2": {"class_type": "EmptyLatentImage", "inputs": {"width": 64}}, + "17": {"class_type": "PrimitiveStringMultiline", "inputs": {"value": "a car"}}, + "14": { + "class_type": "PrimitiveStringMultiline", + "inputs": {"value": "You are a prompt engineer."}, + }, + "11": { + "class_type": "TextGenerate", + "inputs": { + "prompt": ["17", 0], + "system_prompt": ["14", 0], + "clip": ["1", 1], + "max_length": 4096, + "sampling_mode.seed": 0, + }, + }, + "36": { + "class_type": "PromptManagerPair", + "inputs": { + "text": ["11", 0], + "negative_text": "", + "prepend_text": "", + "append_text": "", + "clip": ["1", 1], + }, + }, + "7": { + "class_type": "KSampler", + "inputs": { + "seed": 1, + "steps": 20, + "cfg": 1.0, + "sampler_name": "euler", + "scheduler": "simple", + "denoise": 1.0, + "model": ["1", 0], + "positive": ["36", 0], + "negative": ["36", 1], + "latent_image": ["2", 0], + }, + }, + "8": { + "class_type": "VAEDecode", + "inputs": {"samples": ["7", 0], "vae": ["1", 2]}, + }, + "9": { + "class_type": "SaveImage", + "inputs": {"filename_prefix": "moss", "images": ["8", 0]}, + }, + } + + +class TestRecordedTexts(unittest.TestCase): + """The texts PromptManager recorded at execution, per sampler of the run.""" + + def test_a_run_time_text_shows_what_the_run_recorded(self): + alone = _evidence(text_generate_run(), file_name="moss_00026_.png") + self.assertEqual(alone["resolution"]["reason_code"], "runtime-value") + evidence = _evidence( + text_generate_run(), + file_name="moss_00026_.png", + recorded_saver_id="9", + recorded_texts={"7": {"positive": LLM_TEXT, "negative": ""}}, + ) + self.assertEqual( + evidence["prompts"], + { + "positive": { + "status": "resolved", + "text": LLM_TEXT, + "source": "execution", + }, + "negative": {"status": "resolved", "text": "", "source": "execution"}, + }, + ) + self.assertEqual(evidence["resolution"]["state"], "resolved") + self.assertEqual(evidence["resolution"]["sampler_id"], "7") + + def test_an_unrecorded_negative_stays_open(self): + evidence = _evidence( + text_generate_run(), + recorded_texts={"7": {"positive": LLM_TEXT, "negative": None}}, + ) + self.assertEqual(evidence["prompts"]["positive"]["text"], LLM_TEXT) + self.assertEqual(evidence["prompts"]["negative"]["status"], "unresolved") + + def test_texts_that_do_not_name_the_open_sampler_change_nothing(self): + alone = _evidence(text_generate_run()) + for texts in ( + {"8": {"positive": LLM_TEXT, "negative": ""}}, + {"7": {"positive": None, "negative": ""}}, + {"7": {"positive": " ", "negative": ""}}, + {"7": "not roles"}, + {}, + ): + with self.subTest(texts=texts): + self.assertEqual( + _evidence(text_generate_run(), recorded_texts=texts), alone + ) + + def test_recorded_texts_never_replace_a_resolved_or_several_outcome(self): + texts = {"21": {"positive": LLM_TEXT, "negative": ""}} + resolved = _evidence(shapes.branch_bench(), file_name="s45_branch_2_00001_.png") + self.assertEqual( + _evidence( + shapes.branch_bench(), + file_name="s45_branch_2_00001_.png", + recorded_texts=texts, + ), + resolved, + ) + several = _evidence(shapes.branch_bench()) + self.assertEqual( + _evidence(shapes.branch_bench(), recorded_texts=texts), several + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_run_attribution.py b/tests/test_run_attribution.py index ce3c60c3..32f925ca 100644 --- a/tests/test_run_attribution.py +++ b/tests/test_run_attribution.py @@ -9,13 +9,23 @@ import unittest from pathlib import Path from unittest import mock -from database.run_attribution import stored_run_prompt +from database.run_attribution import ( + RecordedOutput, + recorded_output, + recorded_output_node, + stored_run_prompt, +) from tests import test_execution_history_database as history_tests from tests.background_stops import close_handler from tests.test_api_admin import AdminAPITestCase -from tests.test_execution_context_database import snapshot +from tests.test_execution_context_database import known, snapshot from tests.test_prompt_attribution import issue_graph, llm_text_graph, make_tracker -from utils.execution_contexts import ContextBundle, RecipeContext +from utils.execution_contexts import ( + ContextBundle, + PromptSnapshot, + RecipeContext, + TextEvidence, +) from utils.execution_outputs import ( OutputBundle, OutputReference, @@ -38,28 +48,34 @@ class RecordedRuns: start = history_tests.HistoryDatabaseTests.start finish = history_tests.HistoryDatabaseTests.finish - def record_run(self, db, output_root, files, positives, item=0, window=None): + def record_run( + self, db, output_root, files, positives, item=0, window=None, nodes=None + ): started, finished = window or (now_ms() - 5000, now_ms() + 5000) start = self.start(db, item, started) db.write_execution_start(start) finish = self.finish(start, time=finished) db.write_execution_finish(finish) - db.write_execution_contexts( - start.run_id, - start.storage_epoch, - ContextBundle( - tuple( - RecipeContext(str(4 + i), snapshot(text), snapshot(""), "inferred") - for i, text in enumerate(positives) + if positives: # a run with no PromptManager node records no contexts + db.write_execution_contexts( + start.run_id, + start.storage_epoch, + ContextBundle( + tuple( + RecipeContext( + str(4 + i), snapshot(text), snapshot(""), "inferred" + ) + for i, text in enumerate(positives) + ), + "complete", + "captured", ), - "complete", - "captured", - ), - ) + ) root = snapshot_output_roots(STYLE, output_root, output_root).output_key + nodes = nodes or ["9"] * len(files) references = tuple( - OutputReference("9", i, "output", sub, name, STYLE, root) - for i, (sub, name) in enumerate(files) + OutputReference(node, i, "output", sub, name, STYLE, root) + for i, (node, (sub, name)) in enumerate(zip(nodes, files)) ) db.write_execution_outputs_for_finish( finish, OutputBundle(references, "complete", "reported") @@ -171,6 +187,163 @@ class TestStoredRunPrompt(RecordedRuns, unittest.TestCase): self.assertEqual(linked, [prompt_id]) +class TestRecordedOutputNode(RecordedRuns, unittest.TestCase): + """#386: the save node history says wrote a file, under the same rules.""" + + close_databases = history_tests.HistoryDatabaseTests.close_databases + database = history_tests.HistoryDatabaseTests.database + setUp = TestStoredRunPrompt.setUp + + def node(self, path=None): + return recorded_output_node(self.db, path or self.image, self.output) + + def test_one_run_one_saver_names_the_saver(self): + other = os.path.join(self.output, "sub", "img_00002_.png") + Path(other).write_bytes(b"png") + self.record_run( + self.db, + self.output, + [("sub", "img_00001_.png"), ("sub", "img_00002_.png")], + [], + nodes=["13", "23"], + ) + self.assertEqual(self.node(), "13") + self.assertEqual(self.node(other), "23") + + def test_a_run_without_prompt_manager_contexts_still_names_the_saver(self): + self.record_run(self.db, self.output, [("sub", "img_00001_.png")], []) + self.assertEqual(self.node(), "9") + + def test_two_runs_reporting_the_file_name_nothing(self): + self.record_run( + self.db, self.output, [("sub", "img_00001_.png")], [], nodes=["13"] + ) + self.record_run( + self.db, self.output, [("sub", "img_00001_.png")], [], item=1, nodes=["23"] + ) + self.assertIsNone(self.node()) + + def test_one_run_reporting_the_file_from_two_savers_names_nothing(self): + self.record_run( + self.db, + self.output, + [("sub", "img_00001_.png"), ("sub", "img_00001_.png")], + [], + nodes=["13", "23"], + ) + self.assertIsNone(self.node()) + + def test_a_file_written_outside_the_run_names_nothing(self): + self.record_run( + self.db, + self.output, + [("sub", "img_00001_.png")], + [], + window=(1_000_000, 2_000_000), + ) + self.assertIsNone(self.node()) + + def test_unrecorded_outside_or_unknown_roots_name_nothing(self): + self.record_run(self.db, self.output, [("sub", "img_00001_.png")], []) + other = os.path.join(self.output, "sub", "other.png") + Path(other).write_bytes(b"png") + self.assertIsNone(self.node(other)) + self.assertIsNone(recorded_output_node(self.db, self.image, None)) + self.assertIsNone(recorded_output_node(self.db, "", self.output)) + + def test_the_run_texts_recorded_at_execution_come_with_the_saver(self): + self.record_run( + self.db, self.output, [("sub", "img_00001_.png")], ["an llm prompt"] + ) + self.assertEqual( + recorded_output(self.db, self.image, self.output), + RecordedOutput("9", {"4": {"positive": "an llm prompt", "negative": ""}}), + ) + + def test_texts_only_inferred_from_the_graph_are_not_recorded(self): + start = self.start(self.db, 0, now_ms() - 5000) + self.db.write_execution_start(start) + finish = self.finish(start, time=now_ms() + 5000) + self.db.write_execution_finish(finish) + inferred = TextEvidence("graph text", "inferred", "graph_literal", None) + self.db.write_execution_contexts( + start.run_id, + start.storage_epoch, + ContextBundle( + ( + RecipeContext( + "7", + PromptSnapshot(inferred, inferred, known(""), known("")), + snapshot("kept"), + "inferred", + ), + ), + "complete", + "captured", + ), + ) + root = snapshot_output_roots(STYLE, self.output, self.output).output_key + self.db.write_execution_outputs_for_finish( + finish, + OutputBundle( + ( + OutputReference( + "9", 0, "output", "sub", "img_00001_.png", STYLE, root + ), + ), + "complete", + "reported", + ), + ) + self.assertEqual( + recorded_output(self.db, self.image, self.output).texts, + {"7": {"positive": None, "negative": "kept"}}, + ) + + def test_several_savers_name_no_saver_but_keep_the_texts(self): + self.record_run( + self.db, + self.output, + [("sub", "img_00001_.png"), ("sub", "img_00001_.png")], + ["text"], + nodes=["13", "23"], + ) + found = recorded_output(self.db, self.image, self.output) + self.assertIsNone(found.saver_id) + self.assertEqual(found.texts, {"4": {"positive": "text", "negative": ""}}) + + def test_the_route_lookup_logs_a_failure_and_names_nothing(self): + from database.run_attribution import recorded_output_lookup + + broken = mock.Mock() + broken.model.get_connection.return_value.execute.side_effect = ( + sqlite3.OperationalError("locked") + ) + broken.get_execution_contexts.side_effect = sqlite3.OperationalError("x") + with mock.patch( + "database.run_attribution.host_output_directory", return_value=self.output + ): + self.assertIsNone(recorded_output_lookup(broken)(self.image)) + with ( + mock.patch( + "database.run_attribution._recorded_runs", + side_effect=sqlite3.OperationalError("locked"), + ), + self.assertLogs("prompt_manager.run_attribution", "WARNING"), + ): + self.assertIsNone( + recorded_output_lookup(broken, close=True)(self.image) + ) + broken.model.close.assert_called_once_with() + + def test_a_library_without_history_tables_names_nothing(self): + broken = mock.Mock() + broken.model.get_connection.return_value.execute.side_effect = ( + sqlite3.OperationalError("no such table") + ) + self.assertIsNone(recorded_output_node(broken, self.image, self.output)) + + class TestScanUsesHistory(RecordedRuns, AdminAPITestCase): """The output scan applies the same rules as the monitor.""" diff --git a/utils/prompt_inspector.py b/utils/prompt_inspector.py index 0307b5ca..1e0ed6ed 100644 --- a/utils/prompt_inspector.py +++ b/utils/prompt_inspector.py @@ -10,6 +10,7 @@ import threading import zlib from dataclasses import dataclass, field as dataclass_field from pathlib import Path +from collections.abc import Mapping from typing import Any from PIL import Image @@ -461,6 +462,14 @@ def read_inspection_metadata(path: Path) -> dict[str, str]: return read_inspection_file(path).metadata +# A side whose branch recorded run history named (#386): the save node that +# wrote the file picked the sampler; the text is still that branch's graph walk. +RECORDED_RUN_SOURCE = "recorded-run" +RECORDED_TEXT_REASON = "PromptManager recorded this text when the run executed" +# Outcomes the file alone leaves open; only these ask recorded history. +_UNDECIDED = frozenset({"ambiguous", "unresolved"}) + + def _side(status: str, text: str | None = None, source: str | None = None) -> dict: return {"status": status, "text": text, "source": source} @@ -529,15 +538,60 @@ def _parsed_workflow(value: Any) -> dict | None: return value if isinstance(value, dict) else None +def _recorded_execution(resolved: dict, texts: Mapping) -> dict | None: + """Evidence from the texts recorded at one sampler of the file's run (#386). + + Only for an outcome that names one sampler (a single stage stopped at a + run-time value, such as a language model's output feeding PromptManager) + and only when that sampler's positive text was recorded and is not empty. + """ + sampler = resolved.get("sampler_id") + roles = texts.get(sampler) if isinstance(sampler, str) else None + if not isinstance(roles, Mapping): + return None + positive = roles.get("positive") + if not isinstance(positive, str) or not positive.strip(): + return None + negative = roles.get("negative") + prompts = { + "positive": _side("resolved", positive, "execution"), + "negative": ( + _side("resolved", negative, "execution") + if isinstance(negative, str) + else _side("unresolved") + ), + } + resolution = { + "state": "resolved", + "sampler_id": sampler, + "reason": RECORDED_TEXT_REASON, + "reason_code": None, + "node": None, + } + return {"prompts": prompts, "resolution": resolution} + + def inspect_prompt_evidence( metadata: dict, saved: dict | None = None, *, profile: ResolverProfile = INSPECTION_PROFILE, file_name: str | None = None, + recorded_saver_id: str | None = None, + recorded_texts: Mapping[str, Mapping[str, str | None]] | None = None, ) -> dict: """Keep uncertainty visible; a saved association never replaces file evidence. + ``recorded_saver_id`` is the save node recorded run history says wrote the + file (#386). It is asked only when the file alone leaves the prompt open + (ambiguous or unresolved), and the anchored result is kept only when it + names the positive (resolved, or a guider with no negative); its + graph-inferred sides are then labelled ``recorded-run``. ``recorded_texts`` + (sampler id to the effective texts PromptManager recorded at execution in + that run) is used last: when the prompt is still open at one named sampler + whose positive text was recorded, the recorded texts are shown as + ``execution``. + Inspection routes resolve with ``INSPECTION_PROFILE``: bounded work, not graph size. Ingestion (scan, live monitor, admin import) calls the resolver directly with its default profile. ``resolution`` carries the closed @@ -599,6 +653,22 @@ def inspect_prompt_evidence( # Only to recognize overwritten PromptManager text (R1); never a role. canonical["workflow"] = workflow resolved = resolve_prompt_pair(canonical, profile=profile, file_name=file_name) + recorded = False + if resolved["state"] in _UNDECIDED and recorded_saver_id is not None: + anchored = resolve_prompt_pair( + canonical, + profile=profile, + file_name=file_name, + recorded_saver_id=recorded_saver_id, + ) + if anchored["state"] not in _UNDECIDED and isinstance( + anchored["positive"], dict + ): + resolved, recorded = anchored, True + if resolved["state"] in _UNDECIDED and recorded_texts: + executed = _recorded_execution(resolved, recorded_texts) + if executed is not None: + return executed state = resolved["state"] result = {} for role in ("positive", "negative"): @@ -608,9 +678,10 @@ def inspect_prompt_evidence( and isinstance(side, dict) and (role == "negative" or side["text"].strip()) ): - result[role] = _side( - "resolved", side["text"], side["provenance"]["text_source"] - ) + source = side["provenance"]["text_source"] + if recorded and source == "graph-inferred": + source = RECORDED_RUN_SOURCE + result[role] = _side("resolved", side["text"], source) elif state == "resolved" and isinstance(side, dict): # A resolved pair withholds only an empty positive. result[role] = {**_side("unresolved"), "message": _EMPTY_POSITIVE} diff --git a/utils/prompt_pair_resolver.py b/utils/prompt_pair_resolver.py index b45c948c..1080f0b2 100644 --- a/utils/prompt_pair_resolver.py +++ b/utils/prompt_pair_resolver.py @@ -1611,15 +1611,33 @@ class _GraphResolver: for node_id, node in self.graph.items() if self.writes(node, stem) ] - if len(matches) != 1: - return None - _, inputs = self.node(matches[0]) + except (_Unresolved, _Diverging): + return None + if len(matches) != 1: + return None + return self.saver_walk(matches[0]) + + def recorded_saver_stages(self, node_id: str) -> Optional[list]: + """The candidate samplers of the save node recorded history names (#386). + + The node is walked exactly like a file-name match. A node the graph + lacks, a failing walk or no candidate sampler is ``None``: unanchored. + """ + if not isinstance(self.graph.get(node_id), dict): + return None + self.budget.reset(self.per_candidate_visits) + return self.saver_walk(node_id) + + def saver_walk(self, saver_id: str) -> Optional[list]: + """The listed samplers in one saver's linked inputs' ancestry, or None.""" + try: + _, inputs = self.node(saver_id) values = [ value for name, value in inputs.items() if name != "filename_prefix" and _link(value) ] - found = self.stages_from(matches[0], values) + found = self.stages_from(saver_id, values) except (_Unresolved, _Diverging): return None candidates = [ @@ -2256,6 +2274,7 @@ def resolve_prompt_pair( max_depth: Optional[int] = None, max_text_length: Optional[int] = None, file_name: Optional[str] = None, + recorded_saver_id: Optional[str] = None, ) -> Dict[str, Any]: """Resolve parsed metadata; a present API graph is authoritative over text. @@ -2268,9 +2287,14 @@ def resolve_prompt_pair( ``profile`` selects the bounds; explicit ``max_*`` keywords override it. ``file_name`` (inspection only, unanchored) anchors on the one saver that wrote that file (``_GraphResolver.saver_stages``) when exactly one matches. + ``recorded_saver_id`` (inspection only, unanchored) is the save node that + recorded history says wrote the file (#386); it is tried before the file + name, and a node that yields no candidate sampler anchors nothing. """ profile = _profile_with(profile, max_visits, max_depth, max_text_length) - result = _resolve(metadata, output_node_id, adapters, profile, file_name) + result = _resolve( + metadata, output_node_id, adapters, profile, file_name, recorded_saver_id + ) return _legacy_result(result) if profile.legacy else result @@ -2324,6 +2348,7 @@ def _resolve( adapters: Optional[Mapping[Tuple[str, int], OutputAdapter]], profile: ResolverProfile, file_name: Optional[str] = None, + recorded_saver_id: Optional[str] = None, ) -> Dict[str, Any]: if not isinstance(metadata, dict) or not metadata: return _result("absent", reason="No parsed prompt metadata") @@ -2379,11 +2404,11 @@ def _resolve( "unresolved", reason=str(error), reason_code=error.code, node=error.node ) else: - anchored = ( - resolver.saver_stages(file_name) - if semantics is not None and file_name is not None - else None - ) + anchored = None + if semantics is not None and recorded_saver_id is not None: + anchored = resolver.recorded_saver_stages(str(recorded_saver_id)) + if anchored is None and semantics is not None and file_name is not None: + anchored = resolver.saver_stages(file_name) sampler_ids = anchored or [ node_id for node_id, node in normalized.items() diff --git a/web/js/workspace/file-inspector.mjs b/web/js/workspace/file-inspector.mjs index 9a868a8f..3cdc7461 100644 --- a/web/js/workspace/file-inspector.mjs +++ b/web/js/workspace/file-inspector.mjs @@ -158,8 +158,8 @@ function generationFacts(record, listClass, noteClass) { } // D21: the panel badge names the strongest resolved prompt source; any other source shows no badge. // A prompt that left out a run-time PromptManager part (D125) or a later stage's added conditioning (D126) is still -// inferred from the workflow. -const EVIDENCE_BADGES = [['execution', 'Execution recorded'], ['graph-inferred', 'Workflow inferred'], ['runtime-parts-omitted', 'Workflow inferred'], +// inferred from the workflow. "Recorded run" (#386): recorded run history named the save node that wrote the file. +const EVIDENCE_BADGES = [['execution', 'Execution recorded'], ['recorded-run', 'Recorded run'], ['graph-inferred', 'Workflow inferred'], ['runtime-parts-omitted', 'Workflow inferred'], ['added-conditioning', 'Workflow inferred'], ['runtime-parts-omitted-and-added-conditioning', 'Workflow inferred'], ['embedded-parameters', 'Embedded parameters']]; function evidenceBadge(prompts) { const sources = new Set(Object.values(prompts).filter(role => role?.status === 'resolved').map(role => role.source));