From 83199518dfab627121924822999a193f6316b898 Mon Sep 17 00:00:00 2001 From: Vito Date: Thu, 29 Oct 2026 11:32:53 +0000 Subject: [PATCH 1/7] fix(history): join outer-whitespace raw text to the history of the prompt it saves (#438) (#465) The Library stores text.strip() while a run keeps its exact raw text. The history window, run detail scope id, newest pair run and prompt hints now compare the raw text's stored form (one rule, utils/hashing.py, mirrored in SQLite with the same code points). Hints also keep matching the exact raw text, so older rows that kept outer whitespace stay hinted and protected. --- database/execution_contexts.py | 44 +++++++----- database/history_reads.py | 85 ++++++++++++++++-------- tests/comfyui/execution-history.spec.mjs | 5 ++ tests/test_execution_context_database.py | 53 ++++++++++++++- tests/test_execution_history_hooks.py | 15 +++++ tests/test_hashing.py | 43 ++++++++++++ tests/test_history_reads.py | 68 +++++++++++++++++-- utils/hashing.py | 37 +++++++++++ web/js/workspace/history.mjs | 2 +- 9 files changed, 301 insertions(+), 51 deletions(-) diff --git a/database/execution_contexts.py b/database/execution_contexts.py index 09483221..aeb4b801 100644 --- a/database/execution_contexts.py +++ b/database/execution_contexts.py @@ -7,6 +7,7 @@ from typing import Callable, ContextManager, Iterator try: from ..utils.execution_history_events import RunFinish, encode_command + from ..utils.hashing import stored_prompt_text, stored_prompt_text_sql from ..utils.execution_contexts import ( CAPTURE_REASONS, ContextBundle, @@ -15,6 +16,7 @@ try: ) except ImportError: from utils.execution_history_events import RunFinish, encode_command + from utils.hashing import stored_prompt_text, stored_prompt_text_sql from utils.execution_contexts import ( CAPTURE_REASONS, ContextBundle, @@ -134,7 +136,8 @@ def _insert_exact( def _role_hints(role: str) -> str: return ( - f"SELECT r.{role}_prompt_id, tx.text_value " + f"SELECT r.{role}_prompt_id, tx.text_value, " + f"{stored_prompt_text_sql('tx.text_value')} " "FROM execution_run_contexts r " "JOIN execution_contexts c ON c.context_id = r.context_id " f"JOIN execution_prompt_snapshots s ON s.snapshot_id = c.{role}_snapshot_id " @@ -146,21 +149,21 @@ def _role_hints(role: str) -> str: # Set form of the read-time prompt-hint rule in get_execution_contexts(): # a stored role hint is live only in the current storage epoch, and only when -# its exact raw snapshot text matches exactly one prompt, which is the hinted -# one (_prompt_hint). Evaluated once per statement, never per prompt row. +# exactly one prompt has its raw snapshot text or that text's stored form +# (outer whitespace stripped, #438), and that is the hinted one (_prompt_hint). +# Evaluated once per statement, never per prompt row. # Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH. LIVE_HINT_CTES = ( - "stored_hints(prompt_id, raw) AS (" + "stored_hints(prompt_id, raw, stored) AS (" + _role_hints("positive") + " UNION " + _role_hints("negative") - + "), unique_hint_texts(prompt_id, raw) AS (" - "SELECT MIN(id), text COLLATE BINARY FROM prompts " - "WHERE text COLLATE BINARY IN (SELECT raw FROM stored_hints) " - "GROUP BY text COLLATE BINARY HAVING COUNT(*) = 1" + + "), hint_matches(prompt_id, raw, stored, match_id) AS (" + "SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h " + "JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored)" "), live_hint_prompts(prompt_id) AS (" - "SELECT h.prompt_id FROM stored_hints h JOIN unique_hint_texts u " - "ON u.prompt_id = h.prompt_id AND u.raw = h.raw COLLATE BINARY)" + "SELECT prompt_id FROM hint_matches GROUP BY prompt_id, raw, stored " + "HAVING COUNT(*) = 1 AND MIN(match_id) = prompt_id)" ) LIVE_HINT_PROMPT_IDS_SQL = ( "WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts" @@ -168,10 +171,16 @@ LIVE_HINT_PROMPT_IDS_SQL = ( def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None: + """The one prompt whose text is ``raw`` or the stored form of it (#438). + + The Library stores ``text.strip()``; the exact form keeps hints of older + rows whose stored text kept outer whitespace. + """ if raw is None: return None rows = conn.execute( - "SELECT id FROM prompts WHERE text=? COLLATE BINARY LIMIT 2", (raw,) + "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2", + (raw, stored_prompt_text(raw)), ).fetchall() return rows[0][0] if len(rows) == 1 else None @@ -180,23 +189,26 @@ def _hints( conn: sqlite3.Connection, positive: str | None, negative: str | None ) -> tuple: pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative) + stored = None if negative is None else stored_prompt_text(negative) pair_id = None negative_ambiguous = ( - negative is not None + stored is not None and neg_id is None and conn.execute( - "SELECT 1 FROM prompts WHERE text=? COLLATE BINARY LIMIT 1", (negative,) + "SELECT 1 FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 1", + (negative, stored), ).fetchone() is not None ) - if pos_id is not None and negative is not None and not negative_ambiguous: + if pos_id is not None and stored is not None and not negative_ambiguous: rows = conn.execute( "SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON " "n.id=pp.negative_prompt_id " "WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND " "?='') " - "OR (pp.negative_prompt_id=? AND n.text=? COLLATE BINARY)) LIMIT 2", - (pos_id, negative, neg_id, negative), + "OR (pp.negative_prompt_id=? AND n.text COLLATE BINARY IN (?,?))) " + "LIMIT 2", + (pos_id, stored, neg_id, negative, stored), ).fetchall() if len(rows) == 1: pair_id = rows[0][0] diff --git a/database/history_reads.py b/database/history_reads.py index a96227fd..b803f55e 100644 --- a/database/history_reads.py +++ b/database/history_reads.py @@ -15,6 +15,11 @@ from .execution_history import HistoryEpochChanged from .execution_outputs import _capture, _REFERENCE_COLUMNS from .library import _ELIGIBLE +try: + from ..utils.hashing import stored_prompt_text, stored_prompt_text_sql +except ImportError: + from utils.hashing import stored_prompt_text, stored_prompt_text_sql + RunKey = tuple[int, str, int, str] @@ -100,14 +105,43 @@ _ORDER = ( ) # Stale-hint rows a newest-pair lookup inspects before it reports none. _PAIR_RUN_CANDIDATES = 16 +# The recorded texts of one scope, as a JSON array parameter (_scope_text_ids). +_IN_SCOPE = "IN (SELECT value FROM json_each(?))" _MATCHING = ( "SELECT DISTINCT rc.run_id FROM execution_prompt_snapshots s " "JOIN execution_contexts c ON c.positive_snapshot_id=s.snapshot_id " "JOIN execution_run_contexts rc ON rc.context_id=c.context_id " - "WHERE s.raw_text_id=?" + f"WHERE s.raw_text_id {_IN_SCOPE}" ) +def _scope_text_ids(conn: sqlite3.Connection, scope: str) -> str: + """Every recorded raw text whose stored form is ``scope``, as a JSON array. + + The Library stores ``text.strip()`` while a run keeps its exact raw text, + so a raw text joins the scope of the prompt it saves (#438). The values + are compared, never only their digests. + """ + return json.dumps( + [ + row[0] + for row in conn.execute( + # Only snapshot raw texts: effective and affix texts are skipped + "SELECT t.text_id FROM (SELECT DISTINCT raw_text_id AS text_id " + "FROM execution_prompt_snapshots) s CROSS JOIN " + "execution_context_texts t WHERE t.text_id=s.text_id AND " + f"{stored_prompt_text_sql('t.text_value')}=? COLLATE BINARY", + (scope,), + ) + ] + ) + + +def _scope_text_id(raw: str | None) -> str | None: + """The scope digest of a raw text: that of the text the Library stores.""" + return None if raw is None else content_digest("text", (stored_prompt_text(raw),)) + + def _limit(limit: int) -> None: if type(limit) is not int or not 1 <= limit <= 100: raise ValueError("History limit must be an integer from 1 to 100") @@ -255,7 +289,7 @@ class HistoryReadOperations: if saved is None: return None text = saved["text"] - raw_id = None if text is None else content_digest("text", (text,)) + raw_id = _scope_text_id(text) if after is not None and after.raw_text_id != raw_id: raise ValueError("History position text scope changed") group = { @@ -273,18 +307,11 @@ class HistoryReadOperations: next_position=None, resume_position=None, ) - if ( - raw_id is None - or conn.execute( - "SELECT 1 FROM execution_context_texts WHERE text_id=? " - "AND text_value=? COLLATE BINARY", - (raw_id, text), - ).fetchone() - is None - ): + if raw_id is None: return result + scope = _scope_text_ids(conn, stored_prompt_text(text)) where = f"r.run_id IN ({_MATCHING})" - params = [raw_id] + params = [scope] if after is not None: where += f" AND {_KEY}<=(?,?,?,?)" params.extend(after.upper) @@ -338,9 +365,9 @@ class HistoryReadOperations: "LEFT JOIN execution_prompt_snapshots n " "ON n.snapshot_id=c.negative_snapshot_id " "LEFT JOIN execution_context_texts t ON t.text_id=n.raw_text_id " - f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id=? " + f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id {_IN_SCOPE} " "ORDER BY rc.anchor_id COLLATE BINARY,rc.run_id COLLATE BINARY", - (*by_run, raw_id), + (*by_run, scope), ) summaries = {} evidence_by_context = {} @@ -438,6 +465,7 @@ class HistoryReadOperations: if saved["text"] is None: return {"run": None} epoch = self.get_execution_epoch() + stored = stored_prompt_text(saved["text"]) candidates = conn.execute( "SELECT r.run_id,rc.anchor_id,rc.context_id," "c.positive_snapshot_id,c.negative_snapshot_id " @@ -445,10 +473,11 @@ class HistoryReadOperations: "ON rc.run_id=r.run_id JOIN execution_contexts c " "ON c.context_id=rc.context_id JOIN execution_prompt_snapshots s " "ON s.snapshot_id=c.positive_snapshot_id " - "WHERE s.raw_text_id=? AND rc.pair_id=? AND rc.reference_epoch=? " + f"WHERE s.raw_text_id {_IN_SCOPE} AND rc.pair_id=? " + "AND rc.reference_epoch=? " f"ORDER BY {_ORDER},rc.anchor_id COLLATE BINARY LIMIT ?", ( - content_digest("text", (saved["text"],)), + _scope_text_ids(conn, stored), pair_id, epoch, _PAIR_RUN_CANDIDATES, @@ -457,7 +486,11 @@ class HistoryReadOperations: for row in candidates: positive = _values(conn, row["positive_snapshot_id"]) negative = _values(conn, row["negative_snapshot_id"]) - if positive is None or positive["raw"] != saved["text"]: + if ( + positive is None + or positive["raw"] is None + or stored_prompt_text(positive["raw"]) != stored + ): continue hints = _hints(conn, positive["raw"], negative["raw"] if negative else None) if hints[2] == pair_id: @@ -477,11 +510,9 @@ class HistoryReadOperations: snapshots = { row["anchor_id"]: row for row in conn.execute( - "SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id," - "p.raw_text_id AS positive_raw_text_id " + "SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id " "FROM execution_run_contexts rc JOIN execution_contexts c " - "ON c.context_id=rc.context_id LEFT JOIN execution_prompt_snapshots p " - "ON p.snapshot_id=c.positive_snapshot_id WHERE rc.run_id=?", + "ON c.context_id=rc.context_id WHERE rc.run_id=?", (run_id,), ) } @@ -491,12 +522,12 @@ class HistoryReadOperations: context.update( { key: row[key] - for key in ( - "positive_snapshot_id", - "negative_snapshot_id", - "positive_raw_text_id", - ) - } + for key in ("positive_snapshot_id", "negative_snapshot_id") + }, + # The history scope this context belongs to (#438) + positive_raw_text_id=_scope_text_id( + (context["positive"] or {}).get("raw") + ), ) pair = context["pair_id"] context["navigation"] = { diff --git a/tests/comfyui/execution-history.spec.mjs b/tests/comfyui/execution-history.spec.mjs index 09285e63..eab2b16e 100644 --- a/tests/comfyui/execution-history.spec.mjs +++ b/tests/comfyui/execution-history.spec.mjs @@ -309,6 +309,11 @@ test('fresh registered PM variants preserve exact raw whitespace and known empty expect(value.negative.raw).toBe(''); expect(value.negative.effective).toBe(''); expect(value.evidence.positive.raw).toMatchObject({ provenance: 'recorded', source: 'runtime_return' }); + // #438: the Library stores the stripped text; the run is in that prompt's history + expect(value.positive_prompt_id, 'the run names the prompt the Library saved').toEqual(expect.any(Number)); + const runId = (await readExecutionRuns(execution.databasePath, id)).rows[0].run_id; + const response = await execution.request.get(`/prompt_manager/library/${value.positive_prompt_id}/history`);expect(response.ok()).toBe(true); + expect((await response.json()).items.map(item => item.run_id)).toContain(runId); } }); diff --git a/tests/test_execution_context_database.py b/tests/test_execution_context_database.py index 64f87cd1..b9c7325f 100644 --- a/tests/test_execution_context_database.py +++ b/tests/test_execution_context_database.py @@ -474,6 +474,54 @@ class ContextDatabaseTests(unittest.TestCase): rotate_history_epoch(conn) self.assertEqual(assert_same(), set()) + def test_hints_resolve_the_stored_form_of_outer_whitespace_raw_text(self): + # #438: the Library stores text.strip(); the raw run text keeps its spaces. + db = self.database() + conn = db.model.get_connection() + with conn: + conn.execute( + "INSERT INTO prompts(id,text) VALUES(1,'positive'),(2,'negative')" + ) + conn.execute( + "INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) " + "VALUES(7,1,2)" + ) + start = self.capture(db, 0, bundle(" negative\n", positive="positive\n\t")) + context = db.get_execution_contexts(start.run_id)["contexts"][0] + self.assertEqual(context["positive"]["raw"], "positive\n\t") + self.assertEqual( + ( + context["positive_prompt_id"], + context["negative_prompt_id"], + context["pair_id"], + ), + (1, 2, 7), + ) + live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + self.assertEqual(live, {1, 2}) + stored = conn.execute( + "SELECT positive_prompt_id FROM execution_run_contexts WHERE run_id=?", + (start.run_id,), + ).fetchone()[0] + self.assertEqual(stored, 1) + # A legacy row that kept outer whitespace keeps its exact-text hint and + # with it the recorded-run protection from pruning. + with conn: + conn.execute("INSERT INTO prompts(id,text) VALUES(3,' legacy ')") + legacy = self.capture(db, 1, bundle("negative", positive=" legacy ")) + hinted = db.get_execution_contexts(legacy.run_id)["contexts"][0] + self.assertEqual(hinted["positive_prompt_id"], 3) + live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + self.assertEqual(live, {1, 2, 3}) + self.assertEqual(db.prune_orphaned_prompts(), 0) + # Both forms present make the hint ambiguous, in both rule forms. + with conn: + conn.execute("INSERT INTO prompts(id,text) VALUES(4,'legacy')") + hinted = db.get_execution_contexts(legacy.run_id)["contexts"][0] + self.assertIsNone(hinted["positive_prompt_id"]) + live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + self.assertEqual(live, {1, 2}) + def test_hints_require_exact_unique_raw_text_and_never_refresh(self): db = self.database() conn = db.model.get_connection() @@ -504,12 +552,13 @@ class ContextDatabaseTests(unittest.TestCase): db.get_execution_contexts(fresh.run_id)["contexts"][0]["positive_prompt_id"] ) - def test_null_negative_pair_only_matches_known_exact_empty(self): + def test_null_negative_pair_only_matches_a_known_stored_empty(self): db = self.database() with db.model.get_connection() as conn: conn.execute("INSERT INTO prompts(id,text) VALUES(1,'positive')") conn.execute("INSERT INTO prompt_pairs(id,positive_prompt_id) VALUES(7,1)") - for item, raw, expected in ((0, "", 7), (1, " ", None), (2, None, None)): + # A whitespace-only negative is the stored empty one (#438) + for item, raw, expected in ((0, "", 7), (1, " \n", 7), (2, None, None)): candidate = bundle(negative=raw or "") if raw is None: ctx = candidate.contexts[0] diff --git a/tests/test_execution_history_hooks.py b/tests/test_execution_history_hooks.py index 855c4e91..dbb03655 100644 --- a/tests/test_execution_history_hooks.py +++ b/tests/test_execution_history_hooks.py @@ -916,3 +916,18 @@ class CachedRuntimeHistoryTests(HooksCase): from database.run_attribution import named_prompt self.assertIsNone(named_prompt(self.db, self.rows()[1]["run_id"])) + + +class CachedRuntimeWhitespaceHistoryTests(CachedRuntimeHistoryTests): + """Raw text with outer whitespace joins the history of the prompt it saves (#438). + + The Library stores ``"a lighthouse the LLM wrote"``; the run keeps the exact + frozen raw text, newline included, yet is a tracked run of that prompt. + """ + + TEXT = "\u3000 a lighthouse the LLM wrote\n" + + def test_the_library_stores_the_stripped_text(self): + self.assertEqual( + self.db.get_prompt_by_id(self.prompt_id)["text"], self.TEXT.strip() + ) diff --git a/tests/test_hashing.py b/tests/test_hashing.py index ad02cdfc..a06eec86 100644 --- a/tests/test_hashing.py +++ b/tests/test_hashing.py @@ -16,6 +16,9 @@ from tests.background_stops import stop_tracker # noqa: E402 from utils.hashing import ( generate_prompt_hash, is_duplicate_prompt, + STRIPPED_CODE_POINTS, + stored_prompt_text, + stored_prompt_text_sql, ) ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -115,5 +118,45 @@ class TestGeneratePromptHashEdgeCases(unittest.TestCase): self.assertEqual(len(h), 64) +class TestStoredPromptText(unittest.TestCase): + """SQLite strips exactly the characters Python's str.strip() does (#438).""" + + WHITESPACE = "".join(c for c in map(chr, range(0x110000)) if c.isspace()) + + def sql_strip(self, value): + import sqlite3 + + conn = sqlite3.connect(":memory:") + try: + return conn.execute( + f"SELECT {stored_prompt_text_sql('?')}", (value,) + ).fetchone()[0] + finally: + conn.close() + + def test_code_points_are_exactly_python_whitespace(self): + self.assertEqual("".join(map(chr, STRIPPED_CODE_POINTS)), self.WHITESPACE) + + def test_python_rule_is_the_hash_strip(self): + self.assertEqual(stored_prompt_text(" a lighthouse\n"), "a lighthouse") + self.assertEqual( + generate_prompt_hash(stored_prompt_text(" A b\u3000")), + generate_prompt_hash(" A b\u3000"), + ) + + def test_sql_strips_every_python_whitespace_character(self): + for char in self.WHITESPACE: + with self.subTest(code=hex(ord(char))): + value = f"{char}a{char}b{char}" + self.assertEqual(self.sql_strip(value), stored_prompt_text(value)) + mixed = self.WHITESPACE + "a lighthouse é" + self.WHITESPACE + self.assertEqual(self.sql_strip(mixed), "a lighthouse é") + + def test_sql_keeps_what_python_keeps(self): + for value in ("\u200ba\u200b", "\ufeffa", "a\u0301", "", "x"): + with self.subTest(value=repr(value)): + self.assertEqual(self.sql_strip(value), stored_prompt_text(value)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_history_reads.py b/tests/test_history_reads.py index e9e6a408..c02ea1e8 100644 --- a/tests/test_history_reads.py +++ b/tests/test_history_reads.py @@ -9,7 +9,12 @@ from unittest.mock import patch from database.execution_history import HistoryEpochChanged from database.execution_history_schema import history_schema_error -from utils.execution_contexts import ContextBundle, RecipeContext, TextEvidence +from utils.execution_contexts import ( + ContextBundle, + RecipeContext, + TextEvidence, + content_digest, +) from utils.execution_outputs import OutputBundle from tests.test_execution_context_database import bundle, snapshot from tests.test_execution_output_database import bundle as outputs @@ -146,10 +151,11 @@ class HistoryReadTests(unittest.TestCase): ) self.require_reads(db) with db.recorded_history_snapshot(): - for i in range(1, len(values) + 1): - self.assertEqual( - db.list_history_window(i)["recorded_run_count"], 2 if i == 1 else 1 - ) + # Outer whitespace joins the stored form's scope (#438); case and + # Unicode normalization stay distinct. + expected = (4, 1, 4, 4, 1, 1, 2, 2) + for i, count in enumerate(expected, 1): + self.assertEqual(db.list_history_window(i)["recorded_run_count"], count) self.assertIsNone(db.list_history_window(100)) self.assertIsNone(db.list_history_window(101)) page = db.list_history_window(99) @@ -989,3 +995,55 @@ class HistoryReadTests(unittest.TestCase): db.delete_prompt(saved["positive_prompt_id"]) with db.recorded_history_snapshot(): self.assertIsNone(db.get_newest_pair_run(saved["positive_prompt_id"], pair)) + + def test_outer_whitespace_raw_text_joins_the_stored_prompt_scope(self): + # #438: the Library stores text.strip(); the run keeps its raw text. + db = self.database() + positive = db.save_prompt("a lighthouse") + raws = ("a lighthouse\n", "a lighthouse", "  a lighthouse \t") + starts = [ + self.capture(db, i, bundle(positive=raw), time=100 + i) + for i, raw in enumerate(raws) + ] + self.saved(db, "a lighthouse at dusk", 50) + self.capture(db, 9, bundle(positive="a lighthouse at dusk\n"), time=200) + with db.recorded_history_snapshot(): + page = db.list_history_window(positive) + first = db.list_history_window(positive, limit=2) + rest = db.list_history_window( + positive, limit=2, after=first["next_position"] + ) + details = [db.get_history_run(start.run_id) for start in starts] + self.assertEqual(page["recorded_run_count"], 3) + self.assertEqual(page["group"]["scope_status"], "matched") + self.assertEqual( + page["group"]["raw_text_id"], content_digest("text", ("a lighthouse",)) + ) + self.assertEqual( + [r["run_id"] for r in page["items"]], [s.run_id for s in starts] + ) + self.assertTrue(all(r["matching_anchors"] for r in page["items"])) + self.assertEqual(len(page["context_summaries"]), 3) + self.assertEqual( + [r["run_id"] for r in first["items"] + rest["items"]], + [s.run_id for s in starts[1:]] + [starts[0].run_id], + ) + for raw, detail in zip(raws, details): + context = detail["contexts"][0] + self.assertEqual(context["positive"]["raw"], raw) + self.assertEqual( + context["positive_raw_text_id"], page["group"]["raw_text_id"] + ) + self.assertEqual(context["positive_prompt_id"], positive) + self.assertTrue(context["navigation"]["positive_available"]) + + def test_newest_pair_run_follows_outer_whitespace_raw_text(self): + db = self.database() + pair = db.save_prompt_pair("a lighthouse", "fog") + positive = db.get_prompt_pair(pair)["positive_prompt_id"] + newest = self.capture( + db, 0, bundle(" fog\n", positive="a lighthouse\n"), time=100 + ) + with db.recorded_history_snapshot(): + found = db.get_newest_pair_run(positive, pair) + self.assertEqual(found["run"]["run_id"], newest.run_id) diff --git a/utils/hashing.py b/utils/hashing.py index 9bfa924e..65582f8e 100644 --- a/utils/hashing.py +++ b/utils/hashing.py @@ -65,6 +65,43 @@ def generate_prompt_hash(text: str) -> str: return hashlib.sha256(normalized_text.encode("utf-8")).hexdigest() +def stored_prompt_text(text: str) -> str: + """The text the Library stores for raw node text: outer whitespace removed. + + Every prompt writer stores ``text.strip()`` and the hash above strips too, + so this is the form recorded raw run text is compared in (#438). Case and + inner whitespace stay significant. + """ + return text.strip() + + +# The code points str.strip() removes (every str.isspace() character), for +# SQLite's char(); tests/test_hashing.py pins it to the running Python. +STRIPPED_CODE_POINTS = ( + *range(0x09, 0x0E), + *range(0x1C, 0x21), + 0x85, + 0xA0, + 0x1680, + *range(0x2000, 0x200B), + 0x2028, + 0x2029, + 0x202F, + 0x205F, + 0x3000, +) +_STRIP_CODES = ",".join(map(str, STRIPPED_CODE_POINTS)) + + +def stored_prompt_text_sql(expression: str) -> str: + """``stored_prompt_text`` as an SQLite expression over ``expression``. + + SQLite's one-argument trim() removes spaces only; trim(X, Y) removes every + character of Y, multi-byte ones included, so it strips what Python does. + """ + return f"trim({expression}, char({_STRIP_CODES}))" + + def is_duplicate_prompt(text1: str, text2: str, threshold: float = 0.95) -> bool: """ Check if two prompts are likely duplicates using hash comparison. diff --git a/web/js/workspace/history.mjs b/web/js/workspace/history.mjs index 992b895f..75c458bd 100644 --- a/web/js/workspace/history.mjs +++ b/web/js/workspace/history.mjs @@ -173,7 +173,7 @@ export function mountRecordedHistory(container, {api, positiveId, parentScope, s } count.textContent = window.group ? `${countOf(window.recorded_run_count, 'distinct recorded run')} as of this read · ${window.items.length} loaded positions` : ''; const group = window.group; - scopeText.textContent = group?.scope_status === 'uncapturable_text' ? 'This saved positive cannot be matched within frozen-text capture limits.' : 'Runs are grouped by exact frozen positive raw text, independently of saved pairs. Effective text, prepend/append composition and evidence can differ. Reported output locations do not establish context membership or historical file identity.'; + scopeText.textContent = group?.scope_status === 'uncapturable_text' ? 'This saved positive cannot be matched within frozen-text capture limits.' : 'Runs are grouped by frozen positive raw text that is this saved text apart from outer whitespace, independently of saved pairs. Effective text, prepend/append composition and evidence can differ. Reported output locations do not establish context membership or historical file identity.'; status.setAttribute('role', window.error || next.run.error ? 'alert' : 'status'); status.textContent = next.validating ? 'Revalidating recorded history…' : window.loading ? 'Loading recorded history…' : window.error ? 'Recorded history could not be loaded.' : next.refreshRequired ? 'Recorded evidence changed. Reload before using these actions.' : !items.length ? 'No matching recorded executions.' : window.has_more ? 'Newest bounded window · displayed in chronological order.' : 'All matching recorded runs in this window are loaded.'; if (next.run.error && !next.selection.run_id && !next.refreshRequired) status.textContent += ' The previous recorded selection could not be restored.'; From 8870496ea3622d99ab7cebe2b528c53bc41361a4 Mon Sep 17 00:00:00 2001 From: Vito Date: Thu, 29 Oct 2026 12:00:32 +0000 Subject: [PATCH 2/7] feat(monitor): store certain prompts from workflows without a PromptManager node, with a System switch (#247, PR 2) (#467) * feat(inspector): use recorded run history when the graph alone can't name the prompt (#386) When a file's own graph leaves the prompt open, both inspect routes ask recorded run history about the file (one run reported it and it was written during that run). The save node that run reported it from anchors the walk like a file-name match, so dataset branches that share one saver prefix show their own prompt, labelled "Recorded run". Still open at one sampler whose PromptManager text was recorded at execution (a language model's output), the recorded text shows as "Execution recorded". Results the file resolves alone never change; a failed lookup judges the file alone. * fix(inspector): recorded texts follow history's saver; conflicts anchor nothing (#386) Review follow-ups: the texts PromptManager recorded are looked up at the sampler history's saver leads to, never at a sampler the unanchored walk named; a file name naming a different saver than history anchors nothing; outputs and texts are read in one transaction; "Recorded run" ranks below every workflow-inferred source; unexpected lookup errors log at error level. * fix(history): look up recorded runs by history's own spelling of the file (#386) The history route inspects a case-normalized target on Windows, while history matches file names exactly, so the occurrence's reference names the file. * refactor(inspector): share the inspector's resolve as inspection_resolution inspect_prompt_evidence now calls inspection_resolution, which returns the resolution to use, the anchored result and whether the recorded saver chose it. Ingestion can call the same helper for files the default profile cannot decide (#247). * feat(ingestion): one per-file prompt decision for the scan and the monitor utils/file_prompt.py holds decide_file_prompt, the ladder the output scan runs today plus the inspector's rule (inspection profile, then the recorded saver) before the legacy parser, with a SCAN and a MONITOR policy. Junk text (blank, numbers, LoRA tags only, model file names) is never stored. graph_prompt_text and parse_comfyui_prompt move here; the parser accepts parsed metadata (#247). * feat(ingestion): store a file's prompt in one transaction, one prompt per file database/scanned_prompts.py links a file to the prompt of one text, creating it when new, inside one BEGIN IMMEDIATE transaction: a file linked to another prompt is a conflict, a failed link leaves no prompt behind and two writers store one prompt. pair_ingestion gains file_link_rows, link_conflict and Marks/mark_created; ingest_resolved_pair takes optional marks for the components it creates. recorded_runs and named_prompt in run_attribution become public (#247). * feat(scan): decide each scanned file with the shared ladder (#247) _ingest_scanned_file calls decide_file_prompt (scan policy) and stores the result: pairs through ingest_resolved_pair and positives through store_scanned_prompt, both with the scan marks. Files the default profile leaves undecided now get the inspector's resolve, with the saver recorded history names, before the legacy parser, so each branch of a shared-prefix dataset run links its own prompt. Junk text is never stored, a file linked to another prompt is a conflict instead of a second link, and a failed link leaves no prompt behind. _BatchHistory reads the stored prompt and the recorded saver from one recorded_runs pass. _parse_comfyui_prompt calls the moved parse_comfyui_prompt. TESTING.md documents the decision. * fix(ingestion): keep the file-name saver anchor out of the ladder (#390) The #390 ruling queues a file that only its saver's file name resolves for Needs review, never linking it. inspection_resolution gains name_anchor; the ladder passes False, so such a file falls to the legacy floor in the scan (and to Needs review) and is never stored by the monitor, while the recorded saver still decides. The inspector keeps the name anchor. * feat(config): gallery.monitoring.save_foreign_runs, default on (#247) * fix(ingestion): no joined PromptManager text from the legacy floor; junk keeps its step's status The legacy parser returns prepend + text + append, so its answer is refused when a PromptManager-family node has a prepend or append part (#343). Junk found at the undecided-graph step keeps that step's unlinked status and review, so an ambiguous file is still queued for Needs review. The monitor policy's ambiguous file no longer asks for the recorded run, and the file size may be read lazily. * fix(ingestion): keep the legacy answer that is a PromptManager node's own text The joined-text refusal also refused the #170 parity files whose PromptManager prepend is a link the browser parser does not follow: its answer is then the node's literal text, which is what the node stores. Refuse only an answer that is not such a node's own text. * feat(monitor): store the certain prompt of a foreign workflow's output (#247) A foreign output (no positive PromptManager node) is decided by the shared ladder with the MONITOR policy. Certain files are stored at settle with RUN_MARKS through survive_switch; files only recorded history can decide wait in their own bounded set and are re-checked once after the run's outputs are sealed, with the sealed reference. The setting GalleryConfig.SAVE_FOREIGN_RUNS is read for every file. * fix(scan): one prompt per file for recorded-run links; writes survive a library move The recorded run's stored prompt now links in one transaction with the file's link rows, so a file linked to another prompt is a conflict instead of a second link. The scan's stores run through survive_library_switch. A single file asks history once (a one-file _BatchHistory), the scan passes in_output=False and reads the file size only for files with prompt metadata. A conflict row reads "Kept: already linked to a different prompt". TESTING.md records the rules. * feat(settings): save_foreign_runs through POST /settings, applied live (#247) GET /settings reports settings.save_foreign_runs; a save takes a strict bool, writes gallery.monitoring.save_foreign_runs and publishes it to GalleryConfig in the commit, with no restart field. The diagnostics report lists it. The legacy contract snapshot gains the added key (an addition, no legacy key moves). * feat(system): Save prompts from other workflows in Sources & scans (#247, D147) A host setting row below Watch for new outputs drafts save_foreign_runs and saves with the one Save row; the server applies it at once, so no restart is named. While monitoring is drafted off the row is disabled and says it applies while Watch for new outputs is on. The concept comparison maps the inserted row and its mutation check covers it. * test(comfyui): a workflow of ComfyUI's own nodes creates and links its prompt (#247) The real-host attribution spec gains the foreign case (CLIPTextEncode, KSampler, SaveImage): its prompt is created and linked at settle and a second run links to the same prompt. The fixture route lists the waiting foreign files. TESTING.md records the monitor's foreign path and D147. * test(vendor): the execution matrix counts the #247 real-host case * fix(monitor): read a foreign file's size only when the ladder needs it (#247) Uses PR 1's lazy file size and pins that a foreign file linked to another prompt keeps its one link on the pair and the positive store paths. * fix(ingestion): match a file's link rows under its resolved spelling too The scan stores realpath spellings and the monitor the watcher's raw path, so a file reached through a symlinked or junction output folder could get a second generated_images row. file_link_rows now queries the path and its realpath (each normalized and raw) and dedupes by row id. TESTING.md notes the reverse order is matched only when the raw spelling is the realpath. * fix(monitor): wait for the run unless history answered; monitoring off stops foreign writes (#247) Review fixes: at settle a certain answer ends the wait only when the lookup found a record or the answer is the inspector's or the recorded saver's; the legacy floor waits for the re-check. A record without a certain prompt ends the wait at once. Foreign writes need both save_foreign_runs and monitoring_enabled, read again just before the store. Dropped foreign files are logged outside the lock (evictions as evictions, due ones too), and a drained re-check never drops a newer reference. * fix(system): the foreign row can be set while the monitor still watches; its note describes it (#247, D147) Review fixes: while monitoring is saved off but a source still reads watching until restart (D128), the row stays enabled with its note; the note is the switch's aria-describedby. The browser spec restores save_foreign_runs on the shared server. TESTING.md moves D147 into order and records the monitor's review rules. * test(comfyui): each branch of a shared-prefix foreign run links to its own prompt (#247 CUJ 1) * fix(ingestion): a switched-off PromptManager node behind a pass-through stores nothing (#90) The #90 graph walk follows only prompt-like inputs, so a PromptManagerText with Save to library off reaching the encoder through another input name (easy showAnything.anything, a custom node's value) was invisible to it, and the inspector's resolve stored its text. The ladder now checks the positive every step names: a resolved or inspected positive whose path passes a node that is off, or a legacy answer that is such a node's own text, makes the file not_saved. * fix(ingestion): any switched-off node upstream of a positive stores nothing (#90) The ladder walks every input upstream of each sampler or guider positive (not negative, model, clip, latent or image inputs) for PromptManager nodes whose Save to library is off or wired from an unreadable node, so the sampler-less step, an off node with a prepend part and a PromptManagerPair behind a pass-through no longer store its text. The legacy-floor guard reads a node's resolved text. Every guard reports not_saved for a switch that is off and not_saved_wired for one it cannot read. * fix(ingestion): the switch walk skips negative inputs by #90's own rule The upstream walk treated only names containing "negative" as negative, so an off node on a neg, neg_text or text_neg input blocked an enabled positive. prompt_graph gains is_negative_input (the _NEGATIVE_KEY rule, also used by follows_input) and the walk uses it. The guards accept a node with null inputs and a side whose provenance is not an object. * fix(ingestion): the switch walk also skips any input named with "negative" #90's rule matches only whole underscore-separated words, so negative2, negativeprompt and NegativePrompt counted as positive inputs and an off node wired only to one of them blocked the file. The walk skips those too; follows_input is unchanged. * fix(monitor): a foreign not_saved decision logs its skip and never waits (#247, #90) The foreign path stored nothing for a ladder not_saved decision, but logged it as "No certain prompt" and, when only the legacy answer was a switched-off node's text, kept the file waiting for a seal that could only store nothing. Such a decision now ends the wait at settle, and both settle and the re-check log the opted-out path's own line (a wired switch says so). Tests cover an only-PromptManager node switched off (opted out, not foreign), a switched-off PromptManagerText behind a pass-through into a core CLIPTextEncode, a wired switch, a negative-only switched-off node, recorded history holding the off text, and foreign saving off. * fix(monitor): an undecided not_saved waits for its run's record (#247, #90) A not_saved from the ladder's undecided step is not final before a record exists: the recorded saver may still resolve another branch. The settle lookup now keeps such a file waiting; with the record it stores nothing and logs the opted-out line. An early not_saved (a switched-off node upstream of any sampler's positive) never reaches the lookup and stays final. Tests: the undecided case (literal and wired switch) waits, then skips; a switched-off node feeding one branch of a shared-prefix run never waits and stores nothing, not even the branch the recorded saver names. * docs(system): renumber the foreign-runs switch deviation to D158 (#247) #430 took D147 on main; the Save prompts from other workflows row is D158. * test(monitor): a switched-off node in the recorded saver's branch is never foreign (#247, #90) PR 3's repair skips a file whose recorded saver's branch reads a switched-off PromptManager node. On the monitor the same shape never reaches the foreign path: a literal switch is the opted-out path's and a wired one the node's own run's. Nothing is stored at settle or after the run, even from a recorded run holding the off text. --- TESTING.md | 89 +++- py/api/diagnostics_report.py | 2 + py/api/settings_routes.py | 11 +- py/config.py | 13 + tests/browser/system-concept.spec.mjs | 36 +- .../browser/workspace-system-sources.spec.mjs | 65 +++ tests/comfyui/fixtures/ledger_nodes.py | 11 +- tests/comfyui/image-attribution.spec.mjs | 65 ++- tests/fixtures/legacy_contract_snapshot.json | 1 + tests/js/workspace-system-sources.test.js | 17 + tests/test_api_diagnostics_report.py | 1 + tests/test_config.py | 22 + tests/test_foreign_save_to_library.py | 330 +++++++++++++ tests/test_image_linking.py | 25 +- tests/test_image_monitor_singleton.py | 5 + tests/test_run_attribution.py | 465 ++++++++++++++++++ tests/test_settings_sources.py | 37 ++ tests/vendor/comfyui-config.test.mjs | 6 +- utils/foreign_runs.py | 142 ++++++ utils/image_monitor.py | 242 +++++++-- web/js/workspace/system-sources.mjs | 48 +- 21 files changed, 1572 insertions(+), 61 deletions(-) create mode 100644 tests/test_foreign_save_to_library.py create mode 100644 utils/foreign_runs.py diff --git a/TESTING.md b/TESTING.md index 1f93bf62..89aa2e40 100644 --- a/TESTING.md +++ b/TESTING.md @@ -3271,7 +3271,8 @@ until this browser dismissed the 3.2.0 update notice (`pm_update_notice_dismisse the legacy admin page's key); Later dismisses it without scanning, and Review scan dismisses it as the legacy Scan does. "+ Add source" asks validate-source, then adds the folder to the host draft of `gallery_root_paths`; Remove takes one out; "Watch for -new outputs" (`monitoring_enabled`) and Worker threads (`worker_threads`) are host +new outputs" (`monitoring_enabled`), "Save prompts from other workflows" +(`save_foreign_runs`, D158) and Worker threads (`worker_threads`) are host drafts too, and Save writes them with the revision. A save the server refuses because a scan runs toasts "Another job is running: ." with a link to that job (D57) and keeps the drafts; a save refused for one source says the server's message as the status @@ -3520,7 +3521,7 @@ the connection's `total_changes` is 0 at close and the data reads back unchanged `-shm` bytes are not compared: every WAL reader rewrites its read marks there, so no reader can keep it byte-identical (`test_health_check_never_writes`). -Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107, D109–D110 and D111–D114; D121–D134 assigned). D94, +Recorded deviations (plan D60–D75, D79–D82 renumbered D92–D107, D109–D110 and D111–D114; D121–D134 and D158 assigned). D94, the per-section hand-off panel, was retired by S12 (#329): every section is native. - **D92** (shell): no "Sample configuration" chip. The Save status is truthful: "No @@ -4040,6 +4041,26 @@ the per-section hand-off panel, was retired by S12 (#329): every section is nati Cancel-only review matches the concept review shell` (system-tagging-concept.spec) and the batch tests of workspace-system-tagging.spec; the mutation check renames each grid label in turn. +- **D158** (Sources, #247 D-Q4): the concept has no switch for prompts from workflows + without a PromptManager node; the maintainer chose a visible switch over a config key. + A `.setting-row` in the concept's toggle markup follows "Watch for new outputs": + "Save prompts from other workflows" (`#foreign-runs`, host key `save_foreign_runs`, + config.json `gallery.monitoring.save_foreign_runs`, default on), "New outputs of + workflows without a PromptManager node add their prompt to the library when it is + certain.". It is saved with the revision like the other host rows and applied at once + (the monitor reads it for every file), so a save names no restart. While "Watch for + new outputs" is drafted off, a `p.small` under the description says "Applies while + Watch for new outputs is on." (the switch's `aria-describedby`; the value is kept) and + the switch is disabled, except while a source still reads "watching until restart" + (D128): the running observer still delivers files then, so the value can be set. The + server stores no foreign prompt while `monitoring_enabled` is saved off, whatever the + observer does. + Pinned by `sources matches the concept` (the mapping inserts the row after the + monitoring row), `sources comparison detects mutations` (a missing row and a changed + label are caught), `D158: Save prompts from other workflows saves save_foreign_runs + live; off while monitoring is off (#247)` (keyboard on desktop, a tap on touch) and + `D158: saved off while the monitor still watches until a restart, the foreign row can + still be set (D128)` (workspace-system-sources.spec) and the system-sources node test. - **D109** (Logs, S12, #329): the concept's Logs surface on the live server. The toolbar keeps "Level", Refresh and the download (renamed "Download log": the active log, `/logs/download/prompt_manager.log`, through the api client), with all six minimum @@ -5967,6 +5988,70 @@ Tests: `tests/test_file_prompt.py` (one test per ladder step and junk class), saver, a legacy-linked file left as it is, `_BatchHistory.recorded`) and `tests/test_recorded_saver_anchor.py` `TestInspectionResolution`. +**In the monitor.** An output whose workflow has no positive PromptManager node +(`_find_prompt_from_metadata` says `foreign`) used to be logged and skipped. It now +goes through the same ladder with the `MONITOR` policy (`utils/foreign_runs.py`, the +handler's `_ingest_foreign`, `_recheck_foreign`, `_store_foreign`): + +- **At settle.** The ladder runs without history. Certain (`pair` or `positive`): the + prompt is stored and the file linked at once. `history` (an undecided graph inside + ComfyUI's output folder): the file is registered in its own waiting set, then the + ladder runs again at once with the file's recorded output, because the run may have + been sealed before the file settled. When that lookup finds a record, its answer is + final: certain is stored, anything else is logged "No certain prompt for <file>; + left unlinked" and the file stops waiting (no seal is coming). When it finds none, + only the inspector's or the recorded saver's answer is stored; anything else (the + legacy floor included) keeps waiting for the re-check. Anything else at the first + pass is logged "No certain prompt for <file>; left unlinked". +- **After the run ends.** When the run's outputs are sealed, each waiting foreign file + gets one re-check (`ForeignRecheck(reference)`) with the sealed output reference, + so history is asked with the spelling it recorded (a Windows folder whose case + differs from the disk still links). Not certain: "No certain prompt for <file> + after its run ended; left unlinked". An ambiguous inspector answer is never given + the legacy floor in the monitor (D-Q3); an untraceable one still is. +- **Waiting sets.** `_awaiting_foreign` and `_foreign_due` sit beside the keyless sets + with the same bounds (1000 files, one hour) but are separate, so a large foreign run + never evicts a waiting keyless file. An expired entry is logged "No recorded run + for <file>; left unlinked", an evicted one (waiting or due) "Too many files wait for + their run; <file> left unlinked", both after the lock is released. A due re-check + is dropped only when it was scheduled, never a newer reference for the same file. `awaiting_foreign()` lists them; + `shutdown()` and `close()` clear them, and a closed handler re-checks nothing. +- **Store.** One write through `survive_switch` (rerun after a Move library): a pair + through `ingest_resolved_pair(..., marks=RUN_MARKS)`, a positive through + `store_scanned_prompt_or_raise(..., RUN_MARKS)`. A created prompt carries category + `scanned`, tag `auto-scanned` and the note `Saved from a workflow run: <file>`; a + file linked to another prompt is `conflict` and keeps its link. Logged at info + "Foreign run: <status> prompt <id> for <file> (<source>)"; a failure is logged and + the file stays unlinked. +- **Never.** No tracker FIFO entry is read or consumed, nothing is counted (run count + stays 0, no `last_used_at`), and nothing is queued for Needs review. +- **The setting.** `GalleryConfig.SAVE_FOREIGN_RUNS` (`gallery.monitoring. + save_foreign_runs`, default true, only a bool applies) is read for every file, at + settle, at the re-check and again just before the store, together with + `MONITORING_ENABLED`: either saved off stops foreign writes at once (the running + observer watches until a restart, D128); off logs "No PromptManager prompt in + workflow, skipping image" as before. System › Sources & scans "Save prompts from other workflows" (D158) + saves it through `POST /prompt_manager/settings` with no restart field; the + diagnostics report lists it. + +Tests: `tests/test_run_attribution.py` `TestMonitorForeignRuns` (dataset branches +linked after the seal, a seal before settle, a name two runs reported, the setting +off at settle and at the re-check, a plain workflow linked at settle with no run +count and the tracker queue untouched, the sealed reference's spelling, the separate +bound, expiry, a seal while the pending set is full, shutdown, a failed store, no +record at settle keeps waiting, a record without an answer ends the wait, monitoring +saved off, the setting read before the store, a seal between registering and the +settle lookup, a newer due reference kept, evictions logged, one link per file), +`tests/test_image_linking.py` `TestForeignWorkflows`, `tests/test_image_monitor_singleton.py` +(close drops a foreign re-check), `tests/test_config.py`, +`tests/test_settings_sources.py` `TestSaveForeignRunsSetting`, and the real-host +`tests/comfyui/image-attribution.spec.mjs` case "an output of a workflow without a +PromptManager node creates and links its prompt; a second run reuses it" (ComfyUI's +own `CLIPTextEncode`, `KSampler` and `SaveImage`), and "each branch of a +shared-prefix workflow without a PromptManager node links to its own prompt after the +run" (two branches, one save prefix: the recorded saver names each file's branch). +The fixture route's GET also lists `foreign`, the waiting foreign files. + ## Linked prompt text (#343) A link drives `text` on every PromptManager node, and `negative_text` on the Pair, diff --git a/py/api/diagnostics_report.py b/py/api/diagnostics_report.py index dc2de83c..d5040c7f 100644 --- a/py/api/diagnostics_report.py +++ b/py/api/diagnostics_report.py @@ -213,6 +213,8 @@ def public_config(): "max_image_age_days": GalleryConfig.MAX_IMAGE_AGE_DAYS, "max_concurrent_processing": GalleryConfig.MAX_CONCURRENT_PROCESSING, "metadata_extraction_timeout": GalleryConfig.METADATA_EXTRACTION_TIMEOUT, + # config.json gallery.monitoring.save_foreign_runs (#247) + "save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS, }, "integrations": { "lora_manager_enabled": bool(IntegrationConfig.LORA_MANAGER_ENABLED), diff --git a/py/api/settings_routes.py b/py/api/settings_routes.py index eb3da044..d617286f 100644 --- a/py/api/settings_routes.py +++ b/py/api/settings_routes.py @@ -8,7 +8,8 @@ is shared with the LoRA integration's save. GET also reports the image sources and the monitor (``sources.py``). A save that changes the sources runs as a ``source_change`` registry hold, so it is refused while a record-changing job (a scan) runs. ``monitoring_enabled`` -takes effect at the next restart. ``POST /prompt_manager/settings/validate-source`` +takes effect at the next restart; ``save_foreign_runs`` (#247) at once, since +the monitor reads it for every file. ``POST /prompt_manager/settings/validate-source`` checks one path and writes nothing. """ @@ -43,6 +44,7 @@ _SETTINGS_FILE_KEYS = { "INFINITE_SCROLL": ("web_ui", "infinite_scroll"), "WORKER_THREADS": ("performance", "worker_threads"), "MONITORING_ENABLED": ("gallery", "monitoring", "enabled"), + "SAVE_FOREIGN_RUNS": ("gallery", "monitoring", "save_foreign_runs"), "SAFETY_BACKUP": ("maintenance", "safety_backup"), "THUMBNAIL_FORMAT": ("thumbnails", "format"), "THUMBNAIL_QUALITY": ("thumbnails", "quality"), @@ -58,7 +60,7 @@ _SETTINGS_FILE_KEYS = { # The thumbnail settings a running Convert outdated job fixes (#317). _THUMBNAIL_SETTINGS = ("THUMBNAIL_FORMAT", "THUMBNAIL_QUALITY", "THUMBNAIL_EDGE") # Attributes that live on GalleryConfig; the others on PromptManagerConfig. -_GALLERY_SETTINGS = frozenset({"MONITORING_ENABLED"}) +_GALLERY_SETTINGS = frozenset({"MONITORING_ENABLED", "SAVE_FOREIGN_RUNS"}) # Attributes that live on AutotagConfig, named without the AUTOTAG_ prefix. _AUTOTAG_SETTINGS = frozenset( name for name in _SETTINGS_FILE_KEYS if name.startswith("AUTOTAG_") @@ -264,6 +266,7 @@ class SettingsRoutesMixin: "thumbnail_encoders": encoders, "settings": { "monitoring_enabled": GalleryConfig.MONITORING_ENABLED, + "save_foreign_runs": GalleryConfig.SAVE_FOREIGN_RUNS, "result_timeout": PromptManagerConfig.RESULT_TIMEOUT, "webui_display_mode": PromptManagerConfig.WEBUI_DISPLAY_MODE, "infinite_scroll": PromptManagerConfig.INFINITE_SCROLL, @@ -626,6 +629,10 @@ class SettingsRoutesMixin: if not isinstance(data["monitoring_enabled"], bool): raise ValueError("monitoring_enabled must be true or false") updates["MONITORING_ENABLED"] = data["monitoring_enabled"] + if "save_foreign_runs" in data: + if not isinstance(data["save_foreign_runs"], bool): + raise ValueError("save_foreign_runs must be true or false") + updates["SAVE_FOREIGN_RUNS"] = data["save_foreign_runs"] if "safety_backup" in data: if not isinstance(data["safety_backup"], bool): raise ValueError("safety_backup must be true or false") diff --git a/py/config.py b/py/config.py index 6bbf8ba3..b5ad1e49 100644 --- a/py/config.py +++ b/py/config.py @@ -194,6 +194,8 @@ class GalleryConfig: Attributes: MONITORING_ENABLED (bool): Enable/disable automatic image monitoring + SAVE_FOREIGN_RUNS (bool): The monitor stores the certain prompt of an + output whose workflow has no PromptManager node (#247) MONITORING_DIRECTORIES (List[str]): Directories to monitor for new images SUPPORTED_EXTENSIONS (List[str]): Image file extensions to process PROCESSING_DELAY (float): Delay in seconds before processing new files @@ -210,6 +212,8 @@ class GalleryConfig: # Image monitoring settings MONITORING_ENABLED = True + # Read by the monitor for every file, so a save applies without a restart + SAVE_FOREIGN_RUNS = True MONITORING_DIRECTORIES = [] # Auto-detect if empty SUPPORTED_EXTENSIONS = [".png", ".jpg", ".jpeg", ".webp", ".gif"] PROCESSING_DELAY = 2.0 # Seconds to wait before processing new files @@ -251,6 +255,7 @@ class GalleryConfig: return { "monitoring": { "enabled": cls.MONITORING_ENABLED, + "save_foreign_runs": cls.SAVE_FOREIGN_RUNS, "directories": cls.MONITORING_DIRECTORIES, "extensions": cls.SUPPORTED_EXTENSIONS, "processing_delay": cls.PROCESSING_DELAY, @@ -303,6 +308,14 @@ class GalleryConfig: config_logger.warning( "Ignoring gallery monitoring enabled: expected true or false" ) + if "save_foreign_runs" in monitoring: + if isinstance(monitoring["save_foreign_runs"], bool): + cls.SAVE_FOREIGN_RUNS = monitoring["save_foreign_runs"] + else: + config_logger.warning( + "Ignoring gallery monitoring save_foreign_runs: expected true " + "or false" + ) if "directories" in monitoring: directories = cls._validated_directories(monitoring["directories"]) if directories is not None: diff --git a/tests/browser/system-concept.spec.mjs b/tests/browser/system-concept.spec.mjs index 136cfb58..e4320b90 100644 --- a/tests/browser/system-concept.spec.mjs +++ b/tests/browser/system-concept.spec.mjs @@ -618,10 +618,21 @@ const WATCHED = new Set(['watching', 'stopping']); const REFRESH_FILES = { title: 'Refresh files', description: 'Read every source again to find files added or removed outside ComfyUI.', label: 'Refresh' }; +// D158 (#247): the row production adds directly below "Watch for new outputs", in that row's markup. +const FOREIGN_RUNS = { id: 'foreign-runs', label: 'Save prompts from other workflows', + description: 'New outputs of workflows without a PromptManager node add their prompt to the library when it is certain.' }; + +/** D158: the concept's "Watch for new outputs" row as the inserted "Save prompts from other workflows" row. */ +function foreignRunsRow(monitoringRow, settings) { + return { ...monitoringRow, text: [['label', FOREIGN_RUNS.label, FOREIGN_RUNS.id], ['p', FOREIGN_RUNS.description, null]], + control: { ...monitoringRow.control, id: FOREIGN_RUNS.id, checked: settings.save_foreign_runs } }; +} + /** The concept's Sources with the recorded deviations applied, given the server's real settings and sources. */ function mapSourcesConcept(reference, { settings, sources }) { const [sample] = reference.sources; const scan = sample.actions[0]; + const rowAt = reference.surface.tree.indexOf('div.setting-row'); return { ...reference, // The real sources replace the two samples (without the "sample path" note). D98: each configured one has Remove, @@ -629,13 +640,15 @@ function mapSourcesConcept(reference, { settings, sources }) { sources: sources.effective.map(source => ({ ...sample, title: source.label, description: `${source.path} · ${WATCH_TEXT[source.watch]}`, actions: [{ ...scan, text: WATCHED.has(source.watch) ? 'Scan changes' : 'Scan folder' }, { ...scan, text: 'Remove' }] })), - rows: reference.rows.map(row => { - if (row.control.id === 'monitoring') return { ...row, control: { ...row.control, checked: settings.monitoring_enabled } }; + rows: reference.rows.flatMap(row => { + if (row.control.id === 'monitoring') return [{ ...row, control: { ...row.control, checked: settings.monitoring_enabled } }, + foreignRunsRow(row, settings)]; // D113: the range ends at the server's core count (the concept's fixed 16) and shows the real worker count. - return { ...row, control: { ...row.control, max: String(settings.cpu_count), value: String(settings.worker_threads) } }; + return [{ ...row, control: { ...row.control, max: String(settings.cpu_count), value: String(settings.worker_threads) } }]; }), - // D98: "Refresh files" closes the surface. - surface: { ...reference.surface, tree: [...reference.surface.tree, 'div.operation'] }, + // D158: the inserted row after the first setting row; D98: "Refresh files" closes the surface. + surface: { ...reference.surface, tree: [...reference.surface.tree.slice(0, rowAt + 1), 'div.setting-row', + ...reference.surface.tree.slice(rowAt + 1), 'div.operation'] }, operations: [...reference.operations, { ...reference.operations[0], title: REFRESH_FILES.title, description: REFRESH_FILES.description, actions: [{ ...reference.operations[0].actions[0], text: REFRESH_FILES.label }] }], }; @@ -689,10 +702,12 @@ test('sources comparison detects mutations', async ({ page, request }) => { const expected = mapSourcesConcept(concept, server); const production = await productionSources(page, 'dark', { notice: true }); expect(diffInventory(expected, production)).toEqual([]); - // Unmapped, the concept differs only in the real sources, the D98 operation and the D113 range. + // Unmapped, the concept differs only in the real sources, the D98 operation, the D158 row (which moves the D113 + // range to the third row) and the D113 range. expect(diffInventory(concept, production).map(item => item.path.split('.').slice(0, 2).join('.')) .filter((path, index, all) => all.indexOf(path) === index).sort()) - .toEqual(['operations.2', 'operations.length', 'rows.1', 'sources.0', 'sources.1', 'surface.tree'].sort()); + .toEqual(['operations.2', 'operations.length', 'rows.1', 'rows.2', 'rows.length', 'sources.0', 'sources.1', + 'surface.tree'].sort()); const measure = () => page.evaluate(measureSources, '#workspace-main'); const paths = async () => diffInventory(expected, await measure()).map(item => item.path); // A removed notice and a removed source row. @@ -706,6 +721,13 @@ test('sources comparison detects mutations', async ({ page, request }) => { await page.evaluate(() => { document.querySelector('#workspace-main label[for="monitoring"]').textContent = 'Watch for outputs'; }); expect(await paths()).toEqual(['rows.0.text.0.1']); await page.evaluate(() => { document.querySelector('#workspace-main label[for="monitoring"]').textContent = 'Watch for new outputs'; }); + // D158: the inserted row is compared too: a missing one and a changed label are caught. + const undoForeign = await mutate(page, { remove: ['#workspace-main .setting-row:has(#foreign-runs)'] }); + expect(await paths()).toEqual(expect.arrayContaining(['rows.length', 'surface.tree.length'])); + await undoForeign(); + await page.evaluate(() => { document.querySelector('#workspace-main label[for="foreign-runs"]').textContent = 'Save other prompts'; }); + expect(await paths()).toEqual(['rows.1.text.0.1']); + await page.evaluate(() => { document.querySelector('#workspace-main label[for="foreign-runs"]').textContent = 'Save prompts from other workflows'; }); await page.evaluate(() => { document.querySelector('#workspace-main .panel-heading button').textContent = 'Add source'; }); expect(await paths()).toEqual(['heading.action.text']); await page.evaluate(() => { document.querySelector('#workspace-main .panel-heading button').textContent = '+ Add source'; }); diff --git a/tests/browser/workspace-system-sources.spec.mjs b/tests/browser/workspace-system-sources.spec.mjs index 556ff4e8..74097aa3 100644 --- a/tests/browser/workspace-system-sources.spec.mjs +++ b/tests/browser/workspace-system-sources.spec.mjs @@ -268,6 +268,71 @@ test('Watch for new outputs saves monitoring_enabled; a restart applies it', asy await expect(sourceList(page).locator('.operation p')).toHaveText(['output · watching', 'input · watching']); }); +test('D158: Save prompts from other workflows saves save_foreign_runs live; off while monitoring is off (#247)', async ({ page, request }, testInfo) => { + try { + const layout = await openSources(page); + const rows = layout.locator('section.surface > .setting-row'); + await expect(rows.locator(':scope > div > label')).toHaveText(['Watch for new outputs', 'Save prompts from other workflows', 'Worker threads']); + const foreign = layout.getByRole('switch', { name: 'Save prompts from other workflows', exact: true }); + const foreignRow = rows.filter({ has: page.locator('#foreign-runs') }); + await expect(foreignRow.locator(':scope > div > p')).toHaveText('New outputs of workflows without a PromptManager node add their prompt to the library when it is certain.'); + await expect(foreign).toBeChecked(); + await expect(foreign).toBeEnabled(); + // Keyboard on desktop, a tap on the touch project. + if (testInfo.project.name === 'touch') await foreign.tap(); + else { + await foreign.focus(); + await page.keyboard.press('Space'); + } + await expect(foreign).not.toBeChecked(); + await expect(statusOf(page)).toHaveText('Unsaved changes.'); + const saved = page.waitForResponse(response => response.request().method() === 'POST' && response.url().endsWith(SETTINGS_URL)); + await saveButton(page).click(); + const response = await saved; + expect(response.request().postDataJSON()).toEqual(expect.objectContaining({ save_foreign_runs: false })); + expect((await response.json()).restart_required_fields).toEqual([]); + // Applied live: the monitor reads it for every file, so no restart is named. + await expect(statusOf(page)).toHaveText('Saved to the server configuration.'); + expect((await serverJson(request, SETTINGS_URL)).settings.save_foreign_runs).toBe(false); + // With Watch for new outputs drafted off nothing is watched: the row is disabled and its note says why. + const monitoring = layout.getByRole('switch', { name: 'Watch for new outputs', exact: true }); + await monitoring.uncheck(); + await expect(foreign).toBeDisabled(); + const note = foreignRow.locator('p.small'); + await expect(note).toHaveText('Applies while Watch for new outputs is on.'); + await expect(foreign).toHaveAttribute('aria-describedby', await note.getAttribute('id')); + await monitoring.check(); + await expect(foreign).toBeEnabled(); + await expect(note).toHaveCount(0); + await expect(foreign).not.toHaveAttribute('aria-describedby', /./); + await expect(statusOf(page)).toHaveText('No unsaved changes.'); + } finally { + // The server is shared by the spec's tests: put the default back whatever happened. + await request.post(SETTINGS_URL, { data: { save_foreign_runs: true } }); + } +}); + +test('D158: saved off while the monitor still watches until a restart, the foreign row can still be set (D128)', async ({ page, request }) => { + await expect(await request.post('/__test__/system/monitor', { data: { watching: ['output', 'input'] } })).toBeOK(); + try { + const layout = await openSources(page); + await layout.getByRole('switch', { name: 'Watch for new outputs', exact: true }).uncheck(); + await saveButton(page).click(); + await expect(sourceList(page).locator('.operation p').first()).toHaveText('output · watching until restart'); + const foreign = layout.getByRole('switch', { name: 'Save prompts from other workflows', exact: true }); + await expect(foreign).toBeEnabled(); + await expect(layout.locator('.setting-row').filter({ has: page.locator('#foreign-runs') }).locator('p.small')) + .toHaveText('Applies while Watch for new outputs is on.'); + await foreign.uncheck(); + const saved = page.waitForRequest(item => item.method() === 'POST' && item.url().endsWith(SETTINGS_URL)); + await saveButton(page).click(); + expect((await saved).postDataJSON()).toEqual({ save_foreign_runs: false, expected_revision: expect.any(String) }); + await expect(statusOf(page)).toHaveText('Saved to the server configuration.'); + } finally { + await request.post(SETTINGS_URL, { data: { save_foreign_runs: true } }); + } +}); + test('worker range max is cpu_count', async ({ page, request }) => { const { settings } = await serverJson(request, SETTINGS_URL); const layout = await openSources(page); diff --git a/tests/comfyui/fixtures/ledger_nodes.py b/tests/comfyui/fixtures/ledger_nodes.py index a0c57444..2fcdbca0 100644 --- a/tests/comfyui/fixtures/ledger_nodes.py +++ b/tests/comfyui/fixtures/ledger_nodes.py @@ -334,8 +334,15 @@ async def image_monitor(request: web.Request) -> web.Response: @PromptServer.instance.routes.get("/pm_execution_fixture/monitor") async def image_monitor_state(request: web.Request) -> web.Response: - """File names of keyless images waiting for their run's recorded outputs.""" + """File names of images waiting for their run's recorded outputs: keyless + ones (``awaiting``) and foreign ones (``foreign``, #247).""" monitor = _plugin_module("utils.image_monitor")._monitor_instance handler = getattr(monitor, "handler", None) waiting = handler.awaiting_history() if handler is not None else [] - return web.json_response({"awaiting": [os.path.basename(p) for p in waiting]}) + foreign = handler.awaiting_foreign() if handler is not None else [] + return web.json_response( + { + "awaiting": [os.path.basename(p) for p in waiting], + "foreign": [os.path.basename(p) for p in foreign], + } + ) diff --git a/tests/comfyui/image-attribution.spec.mjs b/tests/comfyui/image-attribution.spec.mjs index 65ffd655..e472e86b 100644 --- a/tests/comfyui/image-attribution.spec.mjs +++ b/tests/comfyui/image-attribution.spec.mjs @@ -1,6 +1,7 @@ // #342 on the real pinned host: the image monitor links an output only to a prompt // that provably produced it. An unknown node (PMHistoryCached) stands in for the -// maintainer's LLM; KSampler is the fixture's deterministic stand-in. +// maintainer's LLM; KSampler is the fixture's deterministic stand-in. #247: an output +// of a workflow without a PromptManager node gets its own prompt. import { randomUUID } from 'node:crypto'; import { execFile } from 'node:child_process'; import { promisify } from 'node:util'; @@ -141,6 +142,68 @@ test('an output saved without metadata links once its recorded run names a store } }); +// #247: no PromptManager node at all, only ComfyUI's own text encoders, sampler and +// saver (the fixture's inputs and decoder stand in for the model files). +function foreignGraph({ text, seed = 1, prefix }) { + return { + '10': { class_type: 'PMHistoryInputs', inputs: {} }, + '20': { class_type: 'CLIPTextEncode', inputs: { text, clip: ['10', 0] } }, + '30': { class_type: 'CLIPTextEncode', inputs: { text: '', clip: ['10', 0] } }, + '40': { class_type: 'KSampler', inputs: { model: ['10', 1], latent_image: ['10', 2], positive: ['20', 0], + negative: ['30', 0], seed, steps: 1, cfg: 1, sampler_name: 'euler', scheduler: 'normal', denoise: 1 } }, + '60': { class_type: 'PMHistoryImages', inputs: { value: 64, batch: 1, latent: ['40', 0] } }, + '70': { class_type: 'SaveImage', inputs: { images: ['60', 0], filename_prefix: prefix } }, + }; +} + +test('an output of a workflow without a PromptManager node creates and links its prompt; a second run reuses it', async ({ execution }, testInfo) => { + const text = `foreign lighthouse at dusk ${testInfo.project.name} ${randomUUID()}`; + const first = await saved(execution.request, foreignGraph({ text, prefix: 'foreign/run' })); + await linkedTo(execution, first, text); + // Only the seed changes: a new file, the same text, so the same stored prompt. + const second = await saved(execution.request, foreignGraph({ text, seed: 2, prefix: 'foreign/run' })); + await linkedTo(execution, second, text); + const state = await readLinks(execution.databasePath, [first, second], [text]); + expect(state).toEqual({ links: { [first]: text, [second]: text }, stored: { [text]: true } }); + // Decided from the graph at settle: nothing waits for the run's recorded outputs. + expect((await (await execution.request.get('/pm_execution_fixture/monitor')).json()).foreign).toEqual([]); +}); + +// CUJ 1: two branches saved under one prefix, so neither the graph nor the file name +// names a file's branch; the run's recorded saver does, once its outputs are sealed. +function sharedPrefixGraph(texts) { + const nodes = { + '10': { class_type: 'PMHistoryInputs', inputs: {} }, + '30': { class_type: 'CLIPTextEncode', inputs: { text: '', clip: ['10', 0] } }, + }; + texts.forEach((text, index) => { + const base = (index + 1) * 100; + nodes[String(base)] = { class_type: 'CLIPTextEncode', inputs: { text, clip: ['10', 0] } }; + nodes[String(base + 1)] = { class_type: 'KSampler', inputs: { model: ['10', 1], latent_image: ['10', 2], + positive: [String(base), 0], negative: ['30', 0], seed: index + 1, steps: 1, cfg: 1, sampler_name: 'euler', + scheduler: 'normal', denoise: 1 } }; + nodes[String(base + 2)] = { class_type: 'PMHistoryImages', inputs: { value: 64, batch: 1, latent: [String(base + 1), 0] } }; + nodes[String(base + 3)] = { class_type: 'SaveImage', inputs: { images: [String(base + 2), 0], filename_prefix: 'foreign/dataset' } }; + }); + return nodes; +} + +test('each branch of a shared-prefix workflow without a PromptManager node links to its own prompt after the run', async ({ execution }, testInfo) => { + const tag = `${testInfo.project.name} ${randomUUID()}`; + const texts = [`dataset branch one harbor ${tag}`, `dataset branch two meadow ${tag}`]; + const id = randomUUID(); + const response = await execution.request.post('/prompt', { data: { prompt: sharedPrefixGraph(texts), prompt_id: id } }); + expect(response.ok()).toBe(true); + let item; + await expect.poll(async () => { + item = (await (await execution.request.get(`/history/${id}`)).json())[id]; + return item?.status?.messages?.some(([kind]) => kind === 'execution_success') || false; + }, { timeout: 15000 }).toBe(true); + const files = ['103', '203'].map(node => item.outputs[node].images[0].filename); + for (const [index, file] of files.entries()) await linkedTo(execution, file, texts[index]); + expect((await (await execution.request.get('/pm_execution_fixture/monitor')).json()).foreign).toEqual([]); +}); + // #332 on the real host: the migration ran at startup and the image category routes // set, list and remove an own category on a generated output. test('an output gets its own image category and loses it again', async ({ execution }, testInfo) => { diff --git a/tests/fixtures/legacy_contract_snapshot.json b/tests/fixtures/legacy_contract_snapshot.json index 62ee7af2..b24dcc86 100644 --- a/tests/fixtures/legacy_contract_snapshot.json +++ b/tests/fixtures/legacy_contract_snapshot.json @@ -192,6 +192,7 @@ "settings.monitoring_enabled", "settings.result_timeout", "settings.safety_backup", + "settings.save_foreign_runs", "settings.thumbnail_edge", "settings.thumbnail_format", "settings.thumbnail_quality", diff --git a/tests/js/workspace-system-sources.test.js b/tests/js/workspace-system-sources.test.js index ad08662c..81719d44 100644 --- a/tests/js/workspace-system-sources.test.js +++ b/tests/js/workspace-system-sources.test.js @@ -157,3 +157,20 @@ test('a valid configured source missing from the effective list is another spell assert.deepEqual(sourceRows({ sources, saved, current: saved }).rows.at(-1), { key: 'output/.', title: 'output', description: 'output/. · same folder as another source', scan: null, remove: 'output/.' }); }); + +test('D158: "Save prompts from other workflows" drafts save_foreign_runs, disabled while monitoring is off (#247)', async () => { + const { FOREIGN_RUNS, foreignRunsState } = await sourcesModule; + assert.deepEqual(FOREIGN_RUNS, { id: 'foreign-runs', key: 'save_foreign_runs', label: 'Save prompts from other workflows', + description: 'New outputs of workflows without a PromptManager node add their prompt to the library when it is certain.', + off: 'Applies while Watch for new outputs is on.' }); + assert.deepEqual(foreignRunsState({ value: true, monitoring: true }), { disabled: false, checked: true, note: null }); + assert.deepEqual(foreignRunsState({ value: false, monitoring: true }), { disabled: false, checked: false, note: null }); + // Monitoring drafted or saved off: nothing is watched, so the row says why it is disabled and keeps its value. + assert.deepEqual(foreignRunsState({ value: true, monitoring: false }), { disabled: true, checked: true, note: FOREIGN_RUNS.off }); + // Not read yet (or the host read failed): disabled without a value, the host rows' own note applies. + assert.deepEqual(foreignRunsState({ value: undefined, monitoring: undefined }), { disabled: true, checked: null, note: null }); + assert.deepEqual(foreignRunsState({ value: undefined, monitoring: false }), { disabled: true, checked: null, note: null }); + // Saved off while the running monitor still watches until the restart (D128 `stopping`): it can still be set. + assert.deepEqual(foreignRunsState({ value: true, monitoring: false, stopping: true }), + { disabled: false, checked: true, note: FOREIGN_RUNS.off }); +}); diff --git a/tests/test_api_diagnostics_report.py b/tests/test_api_diagnostics_report.py index f1422dbb..113d6c42 100644 --- a/tests/test_api_diagnostics_report.py +++ b/tests/test_api_diagnostics_report.py @@ -85,6 +85,7 @@ CONFIG = { "max_image_age_days", "max_concurrent_processing", "metadata_extraction_timeout", + "save_foreign_runs", }, "integrations": {"lora_manager_enabled", "has_civitai_api_key"}, } diff --git a/tests/test_config.py b/tests/test_config.py index d099bd60..f2d9a353 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -59,6 +59,28 @@ class TestGalleryConfig(unittest.TestCase): self.assertFalse(GalleryConfig.MONITORING_ENABLED) self.assertEqual(GalleryConfig.PROCESSING_DELAY, 5.0) + def test_save_foreign_runs_defaults_on_and_only_booleans_apply(self): + """#247: gallery.monitoring.save_foreign_runs, default true.""" + self.assertIs( + GalleryConfig.get_config()["monitoring"]["save_foreign_runs"], True + ) + GalleryConfig.update_config({"monitoring": {"save_foreign_runs": False}}) + self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, False) + for junk in ("true", 1, None): + with self.subTest(value=junk): + with self.assertLogs("prompt_manager.config", "WARNING") as logs: + GalleryConfig.update_config( + {"monitoring": {"save_foreign_runs": junk}} + ) + self.assertIn( + "Ignoring gallery monitoring save_foreign_runs: expected true " + "or false", + "\n".join(logs.output), + ) + self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, False) + GalleryConfig.update_config({"monitoring": {"save_foreign_runs": True}}) + self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, True) + def test_update_tracking(self): GalleryConfig.update_config({"tracking": {"prompt_timeout": 300}}) self.assertEqual(GalleryConfig.PROMPT_TIMEOUT, 300) diff --git a/tests/test_foreign_save_to_library.py b/tests/test_foreign_save_to_library.py new file mode 100644 index 00000000..18b8270f --- /dev/null +++ b/tests/test_foreign_save_to_library.py @@ -0,0 +1,330 @@ +"""#247 PR 2 x #90: the monitor's foreign path never stores switched-off text. + +A file whose run's prompt has Save to library off is the opted-out path's, +not the foreign path's; a foreign graph whose ladder decision is +``not_saved`` (a switched-off node the #90 walk does not reach) stores +nothing at settle, at the re-check after the seal, or from recorded history, +and leaves no wait behind. +""" + +import os +import unittest + +from tests import saver_anchor_graphs as shapes +from tests import test_run_attribution as attribution +from tests import test_save_to_library as save_tests +from tests.test_pair_scan import sampled_graph +from utils.foreign_runs import ForeignRecheck +from utils.hashing import generate_prompt_hash +from py.config import GalleryConfig # after tests: it stubs the host + +LOGGER = "prompt_manager.image_monitor" +OFF_TEXT = "SECRET OFF TEXT" +SKIPPED = "Save to library is off for this run's prompt, skipping image" + +# The class dict, not the class: a TestCase bound here would run again +Fixture = vars(attribution.TestMonitorForeignRuns) + + +def off_behind(middle="MyNode", key="value"): + """A switched-off PromptManagerText reaching a core CLIPTextEncode through + ``middle``'s ``key`` input: the #90 walk does not follow ``key``.""" + graph = save_tests.run_graph() + graph["1"] = { + "class_type": "PromptManagerText", + "inputs": {"text": OFF_TEXT, "save_to_library": False}, + } + graph["7"] = {"class_type": middle, "inputs": {key: ["1", 0]}} + graph["8"] = { + "class_type": "CLIPTextEncode", + "inputs": {"text": ["7", 0], "clip": ["4", 1]}, + } + graph["3"]["inputs"]["positive"] = ["8", 0] + return graph + + +def off_text_unwired(): + """Only history can decide; the legacy answer is a dangling switched-off + node's own text (the ladder says ``not_saved`` once it may ask history).""" + graph = sampled_graph("unused", "blurry") + graph["1"]["inputs"]["text"] = ["30", 0] + graph["30"] = {"class_type": "VendorPromptMaker", "inputs": {"prompt": OFF_TEXT}} + graph["40"] = { + "class_type": "PromptManagerText", + "inputs": {"text": OFF_TEXT, "save_to_library": False}, + } + return graph + + +class TestForeignPathKeepsSaveToLibrary(attribution.RecordedRuns, unittest.TestCase): + setUp = Fixture["setUp"] + close_databases = Fixture["close_databases"] + wait_for = Fixture["wait_for"] + image = Fixture["image"] + linked_text = Fixture["linked_text"] + key = Fixture["key"] + prompt_count = Fixture["prompt_count"] + settled = Fixture["settled"] + + def reference(self, path): + return { + "folder_type": "output", + "subfolder": "dataset", + "filename": os.path.basename(path), + } + + def seal(self, path, positives=(), nodes=("9",)): + """Record one run reporting ``path`` (a new run id for every call).""" + self.runs = getattr(self, "runs", -1) + 1 + self.record_run( + self.db, + self.output, + [("dataset", os.path.basename(path))], + list(positives), + item=self.runs, + nodes=list(nodes), + ) + + def recheck(self, path): + """The re-check the seal schedules, run through the monitor's entry.""" + with self.assertLogs(LOGGER, "INFO") as logs: + self.handler.process_new_image(path, ForeignRecheck(self.reference(path))) + return "\n".join(logs.output) + + def assert_nothing_stored(self, path): + self.assertIsNone(self.linked_text(path)) + self.assertEqual(self.prompt_count(), 0) + self.assertIsNone(self.db.get_prompt_by_hash(generate_prompt_hash(OFF_TEXT))) + self.assertEqual(self.handler.awaiting_foreign(), []) + self.assertEqual(list(self.handler._foreign_due), []) + + def test_q1_the_only_prompt_manager_node_off_is_opted_out_not_foreign(self): + graph = save_tests.run_graph(save=False) + # The switched-off positive is the only PromptManager node + graph["2"] = {"class_type": "CLIPTextEncode", "inputs": {"text": "blurry"}} + sampler_less = { + "1": { + "class_type": "PromptManagerText", + "inputs": {"text": OFF_TEXT, "save_to_library": False}, + } + } + for name, graph in (("sampled", graph), ("sampler-less", sampler_less)): + with self.subTest(name=name): + self.metadata = {"prompt": graph} + state, _ = self.handler._find_prompt_from_metadata(self.metadata) + self.assertEqual(state, "opted_out") + path = self.image(f"IMG_GEN_{name}_00001_.png") + with self.assertLogs(LOGGER, "INFO") as logs: + self.handler.process_new_image(path) + self.assertIn(SKIPPED, "\n".join(logs.output)) + self.seal(path) + self.assertTrue(self.settled()) + self.assertIn(SKIPPED, self.recheck(path)) + self.assert_nothing_stored(path) + + def test_q2_an_off_text_node_behind_a_pass_through_stores_nothing(self): + for middle, key in (("MyNode", "value"), ("easy showAnything", "anything")): + with self.subTest(middle=middle): + self.metadata = {"prompt": off_behind(middle, key)} + state, _ = self.handler._find_prompt_from_metadata(self.metadata) + self.assertEqual(state, "foreign") + path = self.image(f"IMG_GEN_{key}_00001_.png") + with self.assertLogs(LOGGER, "INFO") as logs: + self.handler.process_new_image(path) + self.assertIn(SKIPPED, "\n".join(logs.output)) + self.assert_nothing_stored(path) + self.seal(path) + self.assertTrue(self.settled()) + self.assertIn(SKIPPED, self.recheck(path)) + self.assert_nothing_stored(path) + + def test_q2_a_wired_switch_behind_a_pass_through_stores_nothing(self): + graph = off_behind() + graph["1"]["inputs"]["save_to_library"] = ["20", 0] + graph["20"] = {"class_type": "SomeBoolNode", "inputs": {}} + self.metadata = {"prompt": graph} + path = self.image("IMG_GEN_wired_00001_.png") + wired = "Save to library is wired from a node the image cannot show" + with self.assertLogs(LOGGER, "INFO") as logs: + self.handler.process_new_image(path) + self.assertIn(wired, "\n".join(logs.output)) + self.seal(path) + self.assertTrue(self.settled()) + self.assertIn(wired, self.recheck(path)) + self.assert_nothing_stored(path) + + def test_q3_an_undecided_not_saved_waits_for_the_record_then_skips(self): + """No record yet: a not_saved of the undecided step is not final (a + recorded saver may resolve another branch), so the file waits for its + run; the decision with the record stores nothing and logs the skip.""" + wired = off_text_unwired() + wired["40"]["inputs"]["save_to_library"] = ["20", 0] + wired["20"] = {"class_type": "SomeBoolNode", "inputs": {}} + cases = ( + ("off", off_text_unwired(), SKIPPED), + ("wired", wired, "Save to library is wired from a node"), + ) + for name, graph, line in cases: + with self.subTest(name=name): + self.metadata = {"prompt": graph} + path = self.image(f"vendor_{name}_00001_.png") + self.handler.process_new_image(path) + self.assertIsNone(self.linked_text(path)) + self.assertEqual(self.handler.awaiting_foreign(), [self.key(path)]) + with self.assertLogs(LOGGER, "INFO") as logs: + self.seal(path) + self.assertTrue( + self.wait_for(lambda: not self.handler.awaiting_foreign()) + ) + self.assertTrue(self.settled()) + self.assertIn(line, "\n".join(logs.output)) + self.assert_nothing_stored(path) + + def test_q3_an_early_not_saved_never_waits_even_with_a_saver_branch(self): + """Review: a switched-off node feeding one branch's positive is the + ladder's early, final not_saved (PR 1's #90 rule covers every + sampler's positive), so the file never waits and nothing is stored, + not even the plain branch the recorded saver names.""" + graph = shapes.shared_prefix() + graph["20"]["inputs"]["text"] = ["50", 0] + graph["50"] = { + "class_type": "PromptManagerText", + "inputs": {"text": OFF_TEXT, "save_to_library": False}, + } + self.metadata = {"prompt": graph} + path = self.image("IMG_GEN_00001_.png") + with self.assertLogs(LOGGER, "INFO") as logs: + self.handler.process_new_image(path) + self.assertIn(SKIPPED, "\n".join(logs.output)) + self.assert_nothing_stored(path) + self.seal(path, nodes=("13",)) + self.assertTrue(self.settled()) + self.assertIn(SKIPPED, self.recheck(path)) + self.assert_nothing_stored(path) + self.assertIsNone( + self.db.get_prompt_by_hash(generate_prompt_hash(shapes.BRANCH_TEXTS[0])) + ) + + def test_q3_an_off_node_in_the_recorded_saver_branch_is_never_foreign(self): + """PR 3's parity case: a switched-off node feeds the very branch the + recorded saver names (saver "23" is branch 2). A literal switch is the + opted-out path's; a wired one is the node's own run's (#90), never the + foreign path's. Nothing is stored at settle or after the run, and no + file waits for a foreign re-check.""" + cases = ( + ("off", False, "opted_out", SKIPPED), + ("wired", ["52", 0], "unknown", "No certain prompt for"), + ) + for name, save, state, line in cases: + with self.subTest(name=name): + graph = shapes.shared_prefix() + graph["50"] = { + "class_type": "PromptManagerText", + "inputs": {"text": OFF_TEXT, "save_to_library": save}, + } + graph["20"]["inputs"]["text"] = ["50", 0] + if isinstance(save, list): + graph["52"] = {"class_type": "SomeBoolNode", "inputs": {}} + self.metadata = {"prompt": graph} + found, _ = self.handler._find_prompt_from_metadata(self.metadata) + self.assertEqual(found, state) + path = self.image(f"IMG_GEN_{name}_00001_.png") + with self.assertLogs(LOGGER, "INFO") as logs: + self.handler.process_new_image(path) + self.assertIn(line, "\n".join(logs.output)) + self.assert_nothing_stored(path) + self.seal(path, positives=[OFF_TEXT], nodes=("23",)) + self.assertTrue(self.settled()) + self.handler.process_new_image(path) + self.assert_nothing_stored(path) + self.assertIsNone( + self.db.get_prompt_by_hash( + generate_prompt_hash(shapes.BRANCH_TEXTS[1]) + ) + ) + + def test_q4_an_off_node_on_the_negative_only_still_links_the_positive(self): + positive = "a lighthouse on a cliff at dusk" + plain = sampled_graph(positive, "unused") + plain["2"] = { + "class_type": "PromptManager", + "inputs": {"text": OFF_TEXT, "save_to_library": False}, + } + behind = sampled_graph(positive, "unused") + behind["2"]["inputs"]["text"] = ["7", 0] + behind["7"] = { + "class_type": "PromptManagerText", + "inputs": {"text": OFF_TEXT, "save_to_library": False}, + } + for name, graph in (("direct", plain), ("behind an encoder", behind)): + with self.subTest(name=name): + self.metadata = {"prompt": graph} + state, _ = self.handler._find_prompt_from_metadata(self.metadata) + self.assertEqual(state, "foreign") + path = self.image(f"plain_{len(name)}_00001_.png") + self.handler.process_new_image(path) + self.assertEqual(self.linked_text(path), positive) + self.assertIsNone( + self.db.get_prompt_by_hash(generate_prompt_hash(OFF_TEXT)) + ) + pairs = self.db.list_prompt_pairs()["pairs"] + self.assertEqual(pairs, []) + self.assertEqual(self.handler.awaiting_foreign(), []) + + def test_q5_recorded_history_never_brings_back_switched_off_text(self): + """Even a recorded run whose contexts hold the off text (an older + history row) cannot name it, at settle or at the re-check.""" + cases = {"pass-through": off_behind(), "dangling": off_text_unwired()} + for name, graph in cases.items(): + with self.subTest(name=name): + self.metadata = {"prompt": graph} + path = self.image(f"IMG_GEN_{len(name)}_00001_.png") + self.seal(path, positives=[OFF_TEXT], nodes=("9",)) + self.handler.process_new_image(path) + self.assert_nothing_stored(path) + self.assertIn(SKIPPED, self.recheck(path)) + self.assert_nothing_stored(path) + + def test_q6_with_foreign_saving_off_save_to_library_is_unchanged(self): + GalleryConfig.SAVE_FOREIGN_RUNS = False + opted_out = save_tests.run_graph(save=False) + self.metadata = {"prompt": opted_out} + skipped = self.image("IMG_GEN_off_00001_.png") + with self.assertLogs(LOGGER, "INFO") as logs: + self.handler.process_new_image(skipped) + self.assertIn(SKIPPED, "\n".join(logs.output)) + self.metadata = {"prompt": off_behind()} + foreign = self.image("IMG_GEN_foreign_00001_.png") + with self.assertLogs(LOGGER, "INFO") as logs: + self.handler.process_new_image(foreign) + self.assertIn("No PromptManager prompt in workflow", "\n".join(logs.output)) + for path in (skipped, foreign): + self.assert_nothing_stored(path) + # A switched-on PromptManager positive still links by its metadata + stored = self.db.save_prompt( + text=save_tests.POSITIVE, + prompt_hash=generate_prompt_hash(save_tests.POSITIVE), + ) + self.metadata = {"prompt": save_tests.run_graph(save=True)} + on = self.image("IMG_GEN_on_00001_.png") + self.handler.process_new_image(on) + self.assertEqual(self.linked_text(on), save_tests.POSITIVE) + self.assertEqual( + self.db.get_prompt_by_hash(generate_prompt_hash(save_tests.POSITIVE))["id"], + stored, + ) + + def test_the_shared_prefix_dataset_still_links_after_the_seal(self): + """Regression guard (passes before the fix): the wait still works.""" + self.metadata = {"prompt": shapes.shared_prefix()} + path = self.image("IMG_GEN_00001_.png") + self.handler.process_new_image(path) + self.assertEqual(self.handler.awaiting_foreign(), [self.key(path)]) + self.seal(path, nodes=("13",)) + self.assertTrue(self.wait_for(lambda: self.linked_text(path))) + self.assertTrue(self.settled()) + self.assertEqual(self.linked_text(path), shapes.BRANCH_TEXTS[0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_image_linking.py b/tests/test_image_linking.py index 3cc8586a..7d7d23be 100644 --- a/tests/test_image_linking.py +++ b/tests/test_image_linking.py @@ -172,17 +172,40 @@ class TestForeignWorkflows(LinkingTestCase): self.tracker.set_current_prompt(text, {"prompt_id": prompt_id}) return prompt_id + def _linked_text(self): + row = ( + self.db.model.get_connection() + .execute( + "SELECT p.text FROM generated_images g " + "JOIN prompts p ON p.id = g.prompt_id WHERE g.image_path = ?", + (self.image,), + ) + .fetchone() + ) + return row[0] if row else None + def test_foreign_image_leaves_the_batch_queue_aligned(self): + """#247: it links to its own prompt and never takes a queue entry.""" first = self._queue_batch_item("item one") self.assertIsNone(self._process({"prompt": foreign_graph()})) self.assertEqual(self.linked, []) self.assertEqual(self.tracker.pop_next_prompt()["id"], first) self.assertIsNone(self.tracker.pop_next_prompt()) + self.assertEqual(self._linked_text(), "a cat") + prompt = self.db.get_prompt_by_hash(generate_prompt_hash("a cat")) + self.assertEqual(prompt["run_count"] or 0, 0) - def test_foreign_image_is_not_linked_by_the_later_fallbacks_either(self): + def test_a_foreign_image_is_linked_only_by_the_ladder(self): self._queue_batch_item("item one") # live tracker and recent prompt exist + untraceable = foreign_graph() + untraceable["9"]["inputs"]["text"] = ["30", 0] + untraceable["30"] = {"class_type": "VendorPromptMaker", "inputs": {}} + self.assertIsNone(self._process({"prompt": untraceable})) + self.assertEqual(self.linked, []) + self.assertIsNone(self._linked_text()) self.assertIsNone(self._process({"prompt": foreign_graph()})) self.assertEqual(self.linked, []) + self.assertEqual(self._linked_text(), "a cat") def test_negative_only_workflow_is_foreign(self): self._save("blurry") diff --git a/tests/test_image_monitor_singleton.py b/tests/test_image_monitor_singleton.py index 320c70be..658404c7 100644 --- a/tests/test_image_monitor_singleton.py +++ b/tests/test_image_monitor_singleton.py @@ -597,6 +597,11 @@ class TestHandlerScheduling(unittest.TestCase): self.handler._schedule_recheck(path) self.assertIsNone(self.handler._worker) self.assertEqual(list(self.handler._history_due), []) + # #247: a foreign file's sealed re-check is dropped the same way + self.handler._await_foreign(path) + self.assertEqual(self.handler.awaiting_foreign(), []) + self.handler._schedule_recheck(path, {"filename": "late.png"}) + self.assertEqual(list(self.handler._foreign_due), []) self.assertEqual(self._live_workers() - before, set()) def test_a_sealed_run_starts_a_worker_on_an_open_handler(self): diff --git a/tests/test_run_attribution.py b/tests/test_run_attribution.py index dbc05d53..bb4e69dd 100644 --- a/tests/test_run_attribution.py +++ b/tests/test_run_attribution.py @@ -33,6 +33,7 @@ from utils.execution_outputs import ( snapshot_output_roots, ) from utils.hashing import generate_prompt_hash +from py.config import GalleryConfig from utils.image_monitor import ImageGenerationHandler STYLE = "windows" if os.name == "nt" else "posix" @@ -771,6 +772,470 @@ class TestMonitorHistoryRecheck(RecordedRuns, unittest.TestCase): run_attribution.notify_outputs_sealed("run", []) # no dead callback raises +class TestMonitorForeignRuns(RecordedRuns, unittest.TestCase): + """#247: outputs of workflows without a PromptManager node, in the monitor.""" + + def setUp(self): + history_tests.HistoryDatabaseTests.setUp(self) + self.db = history_tests.HistoryDatabaseTests.database(self) + self.output = str(Path(self.temp.name) / "output") + os.makedirs(os.path.join(self.output, "dataset")) + self.tracker = make_tracker(self.db) + self.handler = ImageGenerationHandler(self.db, self.tracker) + self.handler.processing_delay = 0 + self.handler.settle_interval = 0.01 + self.addCleanup(close_handler, self, self.handler) + self.metadata = {"prompt": shapes.shared_prefix()} + saved = GalleryConfig.SAVE_FOREIGN_RUNS + self.addCleanup(setattr, GalleryConfig, "SAVE_FOREIGN_RUNS", saved) + GalleryConfig.SAVE_FOREIGN_RUNS = True + patches = ( + mock.patch( + "utils.image_monitor.host_output_directory", return_value=self.output + ), + mock.patch( + "database.run_attribution.host_output_directory", + return_value=self.output, + ), + mock.patch.object( + self.handler.metadata_extractor, + "extract_metadata", + side_effect=lambda path: self.metadata, + ), + ) + for patch in patches: + patch.start() + self.addCleanup(patch.stop) + + close_databases = history_tests.HistoryDatabaseTests.close_databases + wait_for = TestMonitorHistoryRecheck.wait_for + + def image(self, name, subfolder="dataset"): + path = os.path.join(self.output, subfolder, name) + Path(path).write_bytes(b"png") + return path + + def linked_text(self, path): + row = ( + self.db.model.get_connection() + .execute( + "SELECT p.text FROM generated_images g " + "JOIN prompts p ON p.id = g.prompt_id WHERE g.image_path = ?", + (str(path),), + ) + .fetchone() + ) + return row[0] if row else None + + def key(self, path): + return self.handler._awaiting_key(path) + + def prompt_count(self): + connection = self.db.model.get_connection() + return connection.execute("SELECT COUNT(*) FROM prompts").fetchone()[0] + + def settled(self): + return self.wait_for(lambda: self.handler.pending_count() == 0) + + def test_each_dataset_file_links_to_its_branch_once_the_run_ends(self): + """CUJ 1: unlinked while the run is open, its own branch after the seal.""" + paths = [self.image(f"IMG_GEN_0000{n}_.png") for n in (1, 2)] + for path in paths: + self.handler.process_new_image(path) + self.assertEqual([self.linked_text(path) for path in paths], [None, None]) + self.assertEqual( + self.handler.awaiting_foreign(), [self.key(path) for path in paths] + ) + self.assertEqual(self.handler.awaiting_history(), []) + self.record_run( + self.db, + self.output, + [("dataset", os.path.basename(path)) for path in paths], + [], + nodes=["13", "23"], + ) + self.assertTrue(self.wait_for(lambda: all(map(self.linked_text, paths)))) + self.assertTrue(self.settled()) + self.assertEqual( + [self.linked_text(path) for path in paths], list(shapes.BRANCH_TEXTS[:2]) + ) + self.assertEqual(self.handler.awaiting_foreign(), []) + prompt = self.db.get_prompt_by_hash( + generate_prompt_hash(shapes.BRANCH_TEXTS[0]) + ) + self.assertEqual(prompt["category"], "scanned") + self.assertIn("Saved from a workflow run: IMG_GEN_00001_.png", prompt["notes"]) + + def test_a_run_sealed_before_the_file_settles_links_it_at_once(self): + path = self.image("IMG_GEN_00003_.png") + self.record_run( + self.db, self.output, [("dataset", "IMG_GEN_00003_.png")], [], nodes=["33"] + ) + self.handler.process_new_image(path) + self.assertEqual(self.linked_text(path), shapes.BRANCH_TEXTS[2]) + self.assertEqual(self.handler.awaiting_foreign(), []) + + def test_a_name_two_runs_reported_stays_unlinked(self): + """CUJ 3: the file was overwritten; no run proves its saver.""" + path = self.image("IMG_GEN_00004_.png") + reported = [("dataset", "IMG_GEN_00004_.png")] + self.record_run(self.db, self.output, reported, [], nodes=["13"]) + self.record_run(self.db, self.output, reported, [], item=1, nodes=["23"]) + self.handler.process_new_image(path) + self.assertIsNone(self.linked_text(path)) + reference = {"folder_type": "output", "subfolder": "dataset"} + reference["filename"] = "IMG_GEN_00004_.png" + with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs: + self.handler._recheck_foreign(path, reference) + self.assertIn( + "No certain prompt for IMG_GEN_00004_.png after its run ended; " + "left unlinked", + "\n".join(logs.output), + ) + self.assertIsNone(self.linked_text(path)) + self.assertEqual(self.prompt_count(), 0) + + def test_the_setting_off_stores_nothing_at_settle_or_after_the_run(self): + """CUJ 4: read for every file, so it applies without a restart.""" + GalleryConfig.SAVE_FOREIGN_RUNS = False + skipped = self.image("IMG_GEN_00005_.png") + with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs: + self.handler.process_new_image(skipped) + self.assertIn( + "No PromptManager prompt in workflow, skipping image", + "\n".join(logs.output), + ) + self.assertEqual(self.handler.awaiting_foreign(), []) + GalleryConfig.SAVE_FOREIGN_RUNS = True + waiting = self.image("IMG_GEN_00006_.png") + self.handler.process_new_image(waiting) + self.assertEqual(self.handler.awaiting_foreign(), [self.key(waiting)]) + GalleryConfig.SAVE_FOREIGN_RUNS = False + self.record_run( + self.db, self.output, [("dataset", "IMG_GEN_00006_.png")], [], nodes=["13"] + ) + self.assertTrue(self.wait_for(lambda: not self.handler.awaiting_foreign())) + self.assertTrue(self.settled()) + self.assertEqual( + [self.linked_text(p) for p in (skipped, waiting)], [None, None] + ) + self.assertEqual(self.prompt_count(), 0) + + def test_a_plain_foreign_workflow_links_at_settle_and_counts_no_run(self): + """CUJ 2: the default profile decides; the tracker queue is untouched.""" + text = "a lighthouse on a cliff at dusk" + self.metadata = {"prompt": sampled_graph(text, "blurry")} + queued = self.db.save_prompt( + text="queued item", prompt_hash=generate_prompt_hash("queued item") + ) + self.tracker.set_current_prompt("queued item", {"prompt_id": queued}) + first, second = self.image("plain_00001_.png"), self.image("plain_00002_.png") + self.handler.process_new_image(first) + self.handler.process_new_image(second) + self.assertEqual( + [self.linked_text(first), self.linked_text(second)], [text, text] + ) + prompt = self.db.get_prompt_by_hash(generate_prompt_hash(text)) + self.assertEqual(prompt["run_count"] or 0, 0) + self.assertIsNone(prompt.get("last_used_at")) + self.assertEqual(self.tracker.pop_next_prompt()["id"], queued) + self.assertEqual(self.handler.awaiting_foreign(), []) + + def test_the_sealed_reference_spells_the_file_as_history_recorded_it(self): + """Windows: the watcher's spelling differs in case from the run's report.""" + Path(self.output, "Dataset").mkdir() + recorded = self.image("IMG_GEN_00007_.png", subfolder="Dataset") + seen = os.path.join(self.output, "dataset", "img_gen_00007_.png") + Path(seen).write_bytes(b"png") + folded = staticmethod(lambda path: os.path.realpath(path).lower()) + with mock.patch.object(ImageGenerationHandler, "_awaiting_key", folded): + self.handler.process_new_image(seen) + self.assertIsNone(self.linked_text(seen)) + self.record_run( + self.db, + self.output, + [("Dataset", os.path.basename(recorded))], + [], + nodes=["43"], + ) + self.assertTrue(self.wait_for(lambda: self.linked_text(seen))) + self.assertTrue(self.settled()) + self.assertEqual(self.linked_text(seen), shapes.BRANCH_TEXTS[3]) + + def test_a_full_foreign_set_never_evicts_a_waiting_keyless_file(self): + from utils import image_monitor + + keyless = self.image("bare_00001_.png", subfolder="") + with mock.patch.object( + self.handler.metadata_extractor, "extract_metadata", return_value=None + ): + self.handler.process_new_image(keyless) + paths = [self.image(f"IMG_GEN_1000{n}_.png") for n in range(3)] + with ( + mock.patch.object(image_monitor, "MAX_AWAITING_HISTORY", 2), + self.assertLogs("prompt_manager.image_monitor", "INFO") as logs, + ): + for path in paths: + self.handler.process_new_image(path) + self.assertEqual( + self.handler.awaiting_foreign(), [self.key(p) for p in paths[1:]] + ) + self.assertEqual(self.handler.awaiting_history(), [self.key(keyless)]) + self.assertIn( + "Too many files wait for their run; IMG_GEN_10000_.png left unlinked", + "\n".join(logs.output), + ) + + def test_an_expired_foreign_entry_is_logged_and_left_unlinked(self): + from utils import image_monitor + + path = self.image("IMG_GEN_00008_.png") + self.handler.process_new_image(path) + later = time.monotonic() + image_monitor.AWAIT_HISTORY_SECONDS + 1 + with ( + mock.patch.object(image_monitor.time, "monotonic", return_value=later), + self.assertLogs("prompt_manager.image_monitor", "INFO") as logs, + ): + self.assertEqual(self.handler.awaiting_foreign(), []) + self.assertIn( + "No recorded run for IMG_GEN_00008_.png; left unlinked", + "\n".join(logs.output), + ) + self.assertIsNone(self.linked_text(path)) + + def test_a_seal_while_the_pending_set_is_full_is_rechecked_later(self): + from utils import image_monitor + + path = self.image("IMG_GEN_00009_.png") + self.handler.process_new_image(path) + blocker = self.image("blocker.png", subfolder="") + release = threading.Event() + original = self.handler.process_new_image + + def slow(image_path, prompt_snapshot=None): + if image_path == blocker: + release.wait(5) + return None + return original(image_path, prompt_snapshot=prompt_snapshot) + + self.handler.process_new_image = slow + with mock.patch.object(image_monitor, "MAX_PENDING_IMAGES", 1): + self.assertTrue(self.handler.schedule(blocker)) + self.assertTrue(self.wait_for(lambda: self.handler.pending_count() == 1)) + self.record_run( + self.db, + self.output, + [("dataset", "IMG_GEN_00009_.png")], + [], + nodes=["13"], + ) + self.assertEqual(list(self.handler._foreign_due), [path]) + release.set() + self.assertTrue(self.wait_for(lambda: self.linked_text(path))) + self.assertTrue(self.settled()) + self.assertEqual(self.linked_text(path), shapes.BRANCH_TEXTS[0]) + + def test_shutdown_forgets_waiting_and_due_foreign_files(self): + path = self.image("IMG_GEN_00010_.png") + self.handler.process_new_image(path) + self.handler._foreign_due[path] = {"filename": "IMG_GEN_00010_.png"} + self.handler.shutdown() + self.assertEqual(self.handler.awaiting_foreign(), []) + self.assertEqual(list(self.handler._foreign_due), []) + + def link_count(self, path): + connection = self.db.model.get_connection() + return connection.execute( + "SELECT COUNT(*) FROM generated_images WHERE image_path = ?", (str(path),) + ).fetchone()[0] + + def test_a_file_linked_to_another_prompt_never_gets_a_second_link(self): + """One prompt per file: the pair path and the positive path both refuse.""" + other = self.db.save_prompt( + "an earlier prompt", generate_prompt_hash("an earlier prompt") + ) + plain = self.image("plain_00004_.png") + self.db.link_image_to_prompt(other, plain) + self.metadata = {"prompt": sampled_graph("a quiet harbor", "blurry")} + with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs: + self.handler.process_new_image(plain) + self.assertIn("Foreign run: conflict", "\n".join(logs.output)) + dataset = self.image("IMG_GEN_00011_.png") + self.db.link_image_to_prompt(other, dataset) + self.metadata = {"prompt": shapes.shared_prefix()} + self.record_run( + self.db, self.output, [("dataset", "IMG_GEN_00011_.png")], [], nodes=["13"] + ) + self.handler.process_new_image(dataset) + for path in (plain, dataset): + self.assertEqual(self.linked_text(path), "an earlier prompt") + self.assertEqual(self.link_count(path), 1) + for text in ("a quiet harbor", shapes.BRANCH_TEXTS[0]): + self.assertIsNone(self.db.get_prompt_by_hash(generate_prompt_hash(text))) + + def test_the_file_size_is_read_only_when_the_ladder_needs_it(self): + """PR 1's lazy size: a foreign file decided without a store reads none.""" + # Foreign (a core encoder only) and no API graph: never, at step 3 + encoder = {"type": "CLIPTextEncode", "inputs": [], "widgets_values": ["x"]} + self.metadata = {"text_encoder_nodes": [encoder]} + path = self.image("bare_00002_.png", subfolder="") + with mock.patch("utils.foreign_runs.os.path.getsize") as size: + self.handler.process_new_image(path) + size.assert_not_called() + + def untraceable(self): + """Unresolved and only the legacy parser names a text (an unknown node).""" + graph = sampled_graph("unused", "blurry") + graph["1"]["inputs"]["text"] = ["30", 0] + graph["30"] = { + "class_type": "VendorPromptMaker", + "inputs": {"prompt": "a misty valley at dawn"}, + } + return {"prompt": graph} + + def test_no_record_at_settle_keeps_the_file_waiting_for_the_legacy_floor(self): + """Review: a lookup that found no record is not the run's answer yet.""" + self.metadata = self.untraceable() + path = self.image("vendor_00001_.png") + self.handler.process_new_image(path) + self.assertIsNone(self.linked_text(path)) + self.assertEqual(self.handler.awaiting_foreign(), [self.key(path)]) + self.record_run(self.db, self.output, [("dataset", "vendor_00001_.png")], []) + self.assertTrue(self.wait_for(lambda: self.linked_text(path))) + self.assertTrue(self.settled()) + self.assertEqual(self.linked_text(path), "a misty valley at dawn") + + def test_a_record_found_at_settle_without_a_certain_prompt_ends_the_wait(self): + """Review: sealed before settle, still ambiguous: no seal is coming.""" + path = self.image("IMG_GEN_00012_.png") + # One run reported it from two savers: a record, but no saver + reported = [("dataset", "IMG_GEN_00012_.png")] * 2 + self.record_run(self.db, self.output, reported, [], nodes=["13", "23"]) + with self.assertLogs("prompt_manager.image_monitor", "INFO") as logs: + self.handler.process_new_image(path) + self.assertIn( + "No certain prompt for IMG_GEN_00012_.png; left unlinked", + "\n".join(logs.output), + ) + self.assertEqual(self.handler.awaiting_foreign(), []) + self.assertIsNone(self.linked_text(path)) + + def test_monitoring_saved_off_stops_foreign_writes_at_settle_and_recheck(self): + """Review: the observer runs until a restart; nothing is written.""" + enabled = GalleryConfig.MONITORING_ENABLED + self.addCleanup(setattr, GalleryConfig, "MONITORING_ENABLED", enabled) + GalleryConfig.MONITORING_ENABLED = False + self.metadata = {"prompt": sampled_graph("a quiet harbor", "blurry")} + plain = self.image("plain_00005_.png") + self.handler.process_new_image(plain) + self.assertIsNone(self.linked_text(plain)) + GalleryConfig.MONITORING_ENABLED = True + self.metadata = {"prompt": shapes.shared_prefix()} + waiting = self.image("IMG_GEN_00013_.png") + self.handler.process_new_image(waiting) + GalleryConfig.MONITORING_ENABLED = False + reference = {"folder_type": "output", "subfolder": "dataset"} + reference["filename"] = "IMG_GEN_00013_.png" + self.record_run( + self.db, self.output, [("dataset", reference["filename"])], [], nodes=["13"] + ) + self.assertTrue(self.wait_for(lambda: not self.handler.awaiting_foreign())) + self.assertTrue(self.settled()) + self.handler._recheck_foreign(waiting, reference) + self.assertIsNone(self.linked_text(waiting)) + self.assertEqual(self.prompt_count(), 0) + + def test_the_setting_is_read_again_just_before_the_store(self): + self.metadata = {"prompt": sampled_graph("a quiet harbor", "blurry")} + path = self.image("plain_00006_.png") + real = __import__("utils.foreign_runs", fromlist=["x"]).decide_foreign + + def decide_then_switch_off(*args, **kwargs): + decision = real(*args, **kwargs) + GalleryConfig.SAVE_FOREIGN_RUNS = False + return decision + + with mock.patch( + "utils.image_monitor.decide_foreign", side_effect=decide_then_switch_off + ): + self.handler.process_new_image(path) + self.assertIsNone(self.linked_text(path)) + self.assertEqual(self.prompt_count(), 0) + + def test_a_seal_between_registering_and_the_settle_lookup_links_once(self): + """Regression guard (passes before the review fixes): one link only.""" + path = self.image("IMG_GEN_00014_.png") + real = __import__("utils.foreign_runs", fromlist=["x"]).history_lookup + + def sealed_first(db, image_path, reference=None): + lookup = real(db, image_path, reference) + + def run_then_look(): + if reference is None: # the settle lookup: the run seals now + self.record_run( + self.db, + self.output, + [("dataset", "IMG_GEN_00014_.png")], + [], + nodes=["13"], + ) + return lookup() + + return run_then_look + + with mock.patch("utils.image_monitor.history_lookup", side_effect=sealed_first): + self.assertTrue(self.handler.schedule(path)) + self.assertTrue(self.wait_for(lambda: self.linked_text(path))) + self.assertTrue(self.settled()) + self.assertEqual(self.linked_text(path), shapes.BRANCH_TEXTS[0]) + self.assertEqual(self.link_count(path), 1) + self.assertEqual(list(self.handler._foreign_due), []) + + def test_a_drained_recheck_never_drops_a_newer_reference(self): + path = self.image("IMG_GEN_00015_.png") + older, newer = {"filename": "old"}, {"filename": "new"} + self.handler._foreign_due[path] = older + real = self.handler.schedule + + def schedule(image_path, snapshot=None): + # The run is sealed again while the older one is scheduled + self.handler._foreign_due[image_path] = newer + return real(image_path, snapshot) + + self.handler.schedule = schedule + with mock.patch.object(self.handler, "process_new_image"): + self.handler._drain_history_due() + self.assertIs(self.handler._foreign_due[path], newer) + + def test_an_evicted_due_recheck_is_logged(self): + from utils import image_monitor + + self.handler.schedule = lambda *args, **kwargs: False + with ( + mock.patch.object(image_monitor, "MAX_AWAITING_HISTORY", 1), + self.assertLogs("prompt_manager.image_monitor", "INFO") as logs, + ): + for name in ("a_00001_.png", "b_00001_.png"): + self.handler._schedule_recheck(self.image(name), {"filename": name}) + self.assertIn( + "Too many files wait for their run; a_00001_.png left unlinked", + "\n".join(logs.output), + ) + + def test_a_failed_store_is_logged_and_leaves_the_file_unlinked(self): + self.metadata = {"prompt": sampled_graph("a quiet harbor", "blurry")} + path = self.image("plain_00003_.png") + with ( + mock.patch( + "utils.foreign_runs.ingest_resolved_pair", + side_effect=sqlite3.OperationalError("locked"), + ), + self.assertLogs("prompt_manager.image_monitor", "ERROR"), + ): + self.handler.process_new_image(path) + self.assertIsNone(self.linked_text(path)) + + class TestBoundedLookup(RecordedRuns, unittest.TestCase): """M5: history lookups are batched, and the history schema is unchanged.""" diff --git a/tests/test_settings_sources.py b/tests/test_settings_sources.py index 3ab7b031..2fda8459 100644 --- a/tests/test_settings_sources.py +++ b/tests/test_settings_sources.py @@ -323,6 +323,43 @@ class TestMonitoringSetting(SourcesTestCase): self.assertIs(GalleryConfig.MONITORING_ENABLED, False) +class TestSaveForeignRunsSetting(SourcesTestCase): + """#247 D-Q4: "Save prompts from other workflows", applied live.""" + + async def asyncSetUp(self): + await super().asyncSetUp() + saved = GalleryConfig.SAVE_FOREIGN_RUNS + self.addCleanup(setattr, GalleryConfig, "SAVE_FOREIGN_RUNS", saved) + GalleryConfig.SAVE_FOREIGN_RUNS = True + + async def test_read_and_write_published_live_without_a_restart(self): + self.assertIs((await self._get())["settings"]["save_foreign_runs"], True) + self._install_monitor(FakeMonitor([self.output_dir])) + status, data = await self._post( + "/prompt_manager/settings", {"save_foreign_runs": False} + ) + self.assertEqual(status, 200, data) + self.assertTrue(data["persisted"]) + self.assertEqual(data["restart_required_fields"], []) + saved = self._read_config() + self.assertIs(saved["gallery"]["monitoring"]["save_foreign_runs"], False) + self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, False) + self.assertIs((await self._get())["settings"]["save_foreign_runs"], False) + + async def test_only_true_or_false_is_accepted(self): + self._write_config({"custom": {"keep": True}}) + before = self._config_bytes() + for value in ("false", 0, None, []): + with self.subTest(value=value): + status, data = await self._post( + "/prompt_manager/settings", {"save_foreign_runs": value} + ) + self.assertEqual(status, 400, data) + self.assertIn("save_foreign_runs must be true or false", data["error"]) + self.assertEqual(self._config_bytes(), before) + self.assertIs(GalleryConfig.SAVE_FOREIGN_RUNS, True) + + class TestSourceChangeHold(SourcesTestCase): async def test_root_save_refused_while_scan_job_runs(self): GalleryConfig.MONITORING_DIRECTORIES = [self._root(self.a)] diff --git a/tests/vendor/comfyui-config.test.mjs b/tests/vendor/comfyui-config.test.mjs index bf5b1c0d..b77e7d8b 100644 --- a/tests/vendor/comfyui-config.test.mjs +++ b/tests/vendor/comfyui-config.test.mjs @@ -36,14 +36,14 @@ test('execution script discovers only the new matrix and preserves all 78 origin // + the fresh-host library location case, + #342 image attribution, + #343 linked text, // + #332 image category routes on a generated output (image-attribution.spec.mjs, 2 projects), // + #347 cached runtime-linked runs: five node kinds and eight one-renderer cases (execution-text-link.spec.mjs, 2 projects), - // + #90 Save to library off (five node types) - assert.match(executions, /Total: 128 tests in 5 files/); + // + #90 Save to library off (five node types), + #247 foreign workflows (image-attribution.spec.mjs, 2 tests x 2 projects) + assert.match(executions, /Total: 132 tests in 5 files/); for (const project of ['classic-start', 'nodes2-start']) { const cases = executions.split('\n').filter(line => line.includes(`[${project}]`)); assert.equal(cases.filter(line => line.includes('execution-history.spec.mjs')).length, 15); assert.equal(cases.filter(line => line.includes('execution-outputs.spec.mjs')).length, 8); assert.equal(cases.filter(line => line.includes('library-location.spec.mjs')).length, 1); - assert.equal(cases.filter(line => line.includes('image-attribution.spec.mjs')).length, 3); + assert.equal(cases.filter(line => line.includes('image-attribution.spec.mjs')).length, 5); assert.equal(cases.filter(line => line.includes('execution-text-link.spec.mjs')).length, 37); } assert.doesNotMatch(executions, /prompt-manager.spec/); diff --git a/utils/foreign_runs.py b/utils/foreign_runs.py new file mode 100644 index 00000000..03bacc05 --- /dev/null +++ b/utils/foreign_runs.py @@ -0,0 +1,142 @@ +"""The monitor's path for outputs of workflows without a PromptManager node (#247). + +A foreign output (no positive PromptManager node in its graph) is decided by +the ladder the output scan uses (``utils.file_prompt.decide_file_prompt``) +with the ``MONITOR`` policy: certain files are stored at once, files only +recorded history can decide wait for their run's outputs to be sealed, and +everything else stays unlinked. This module holds the parts that do not +need the monitor's waiting sets: the setting, the decision call and the +store. The waiting and the re-check stay on ``ImageGenerationHandler``. +""" + +import os +from dataclasses import dataclass +from typing import Any, Callable, Mapping, Optional, Tuple + +from .file_prompt import ( + NOT_SAVED, + NOT_SAVED_WIRED, + PAIR, + POSITIVE, + Decision, + Policy, + decide_file_prompt, +) +from .hashing import generate_prompt_hash +from .usage_tracking import survive_switch + +try: + from ..database.pair_ingestion import RUN_MARKS, ingest_resolved_pair + from ..database.run_attribution import recorded_output_lookup + from ..database.scanned_prompts import store_scanned_prompt_or_raise +except ImportError: + from database.pair_ingestion import RUN_MARKS, ingest_resolved_pair + from database.run_attribution import recorded_output_lookup + from database.scanned_prompts import store_scanned_prompt_or_raise + +# Decisions that name a prompt to store +CERTAIN = (PAIR, POSITIVE) +# Decisions of a run whose prompt has Save to library off, or wired from a +# node the file cannot show (#90): final, nothing is stored, nothing waits +NOT_SAVED_KINDS = (NOT_SAVED, NOT_SAVED_WIRED) +# Sources certain without a recorded run: the graph alone, or its recorded saver +ANCHORED_SOURCES = ("inspector", "recorded-run") + + +@dataclass(frozen=True) +class ForeignRecheck: + """``schedule`` snapshot: the one re-check of a foreign output after its + run's outputs were sealed. ``reference`` is the sealed output reference, + the spelling history recorded (``recorded_output_lookup(reference=)``).""" + + reference: Mapping[str, Any] + + +def save_foreign_runs() -> bool: + """Whether a foreign output may be stored now, read for every file. + + ``GalleryConfig.SAVE_FOREIGN_RUNS`` and ``MONITORING_ENABLED``: a + monitoring switch saved off stops foreign writes at once, although the + running observer watches until a restart. True when the configuration + cannot be imported (the defaults). + """ + try: + from ..py.config import GalleryConfig + except ImportError: + try: + from py.config import GalleryConfig + except ImportError: + return True + return ( + GalleryConfig.SAVE_FOREIGN_RUNS is not False + and GalleryConfig.MONITORING_ENABLED is not False + ) + + +def history_lookup( + db: Any, image_path: str, reference: Optional[Mapping[str, Any]] = None +) -> Callable[[], Any]: + """The ladder's ``lookup`` for one file: its ``RecordedOutput`` or None. + + With ``reference`` the file is spelled as history recorded it. Failures + are logged by ``recorded_output_lookup`` and name nothing. + """ + lookup = recorded_output_lookup( + db, reference=dict(reference) if reference is not None else None + ) + return lambda: lookup(image_path) + + +def decide_foreign( + image_path: str, + metadata: Any, + *, + in_output: bool, + lookup: Optional[Callable[[], Any]], +) -> Decision: + """The ``MONITOR`` ladder for one foreign output (the size is read only + when the ladder needs it).""" + + def file_size() -> Optional[int]: + try: + return os.path.getsize(image_path) + except OSError: + return None + + return decide_file_prompt( + metadata, + file_name=os.path.basename(image_path), + file_size=file_size, + in_output=in_output, + lookup=lookup, + policy=Policy.MONITOR, + ) + + +def store_foreign( + db: Any, image_path: str, decision: Decision +) -> Tuple[str, Optional[int]]: + """Store a certain decision: ``(status, prompt id)``. + + One write through ``survive_switch``, so it runs again on the moved + library when a Move library switched while it waited (#249 S6b). Created + prompts carry ``RUN_MARKS``. A file linked to another prompt is a + ``conflict`` and nothing is written. Database errors are raised. + """ + + def store(): + if decision.kind == PAIR: + result = ingest_resolved_pair( + db, image_path, decision.canonical, decision.resolution, marks=RUN_MARKS + ) + return result["status"], result.get("image_id") + return store_scanned_prompt_or_raise( + db, image_path, decision.canonical, decision.text, RUN_MARKS + ) + + status, linked = survive_switch(db, store) + if decision.kind != PAIR or linked is None: + return status, linked + # The pair ingester reports the image row; its prompt is the positive's. + prompt = db.get_prompt_by_hash(generate_prompt_hash(decision.text)) + return status, prompt["id"] if prompt else None diff --git a/utils/image_monitor.py b/utils/image_monitor.py index 78cfcb10..d2fc1c19 100644 --- a/utils/image_monitor.py +++ b/utils/image_monitor.py @@ -33,6 +33,17 @@ from typing import Callable, Optional, Dict, Any, List, Tuple from watchdog.observers import Observer from watchdog.events import FileSystemEventHandler +from .file_prompt import HISTORY, NOT_SAVED, NOT_SAVED_WIRED +from .foreign_runs import ( + ANCHORED_SOURCES, + CERTAIN, + NOT_SAVED_KINDS, + ForeignRecheck, + decide_foreign, + history_lookup, + save_foreign_runs, + store_foreign, +) from .hashing import generate_prompt_hash from .metadata_extractor import ComfyUIMetadataExtractor from .parallel import note_refusal, refused_at_shutdown @@ -153,6 +164,11 @@ class ImageGenerationHandler(FileSystemEventHandler): # Sealed re-checks ``schedule`` refused (the path still pending, or the # pending set full): retried each time a pending image finishes. self._history_due: "OrderedDict[str, None]" = OrderedDict() + # The same two sets for foreign outputs (#247), apart so a large + # foreign run never evicts a waiting keyless file; the due set keeps + # each file's sealed output reference. + self._awaiting_foreign: "OrderedDict[str, Tuple[str, float]]" = OrderedDict() + self._foreign_due: "OrderedDict[str, dict]" = OrderedDict() self._awaiting_lock = threading.Lock() self._sealed_subscribed = False # Set by ``close``: no new work, no new worker, ever (#302). @@ -253,6 +269,8 @@ class ImageGenerationHandler(FileSystemEventHandler): with self._awaiting_lock: self._awaiting.clear() self._history_due.clear() + self._awaiting_foreign.clear() + self._foreign_due.clear() with self._pending_lock: self._pending.clear() worker, self._worker = self._worker, None @@ -438,6 +456,9 @@ class ImageGenerationHandler(FileSystemEventHandler): if prompt_snapshot is HISTORY_ONLY: self._recheck_history(image_path) return + if isinstance(prompt_snapshot, ForeignRecheck): + self._recheck_foreign(image_path, prompt_snapshot.reference) + return self.logger.info(f"Processing image: {image_path}") if not os.path.exists(image_path): @@ -457,17 +478,12 @@ class ImageGenerationHandler(FileSystemEventHandler): # negative-prompt PromptManager nodes in the workflow. state, current_prompt = self._find_prompt_from_metadata(metadata) if state == "opted_out": - self.logger.info( - "Save to library is off for this run's prompt, skipping image: " - f"{image_path}" - ) + self._log_not_saved(image_path) return if state == "foreign": - # Another workflow saved this image: it belongs to no prompt of - # ours, and must not consume a batch queue entry. - self.logger.info( - f"No PromptManager prompt in workflow, skipping image: {image_path}" - ) + # Another workflow saved this image: never a batch queue entry + # of ours; its own graph may name its prompt (#247). + self._ingest_foreign(image_path, metadata) return if current_prompt: self.logger.info( @@ -589,36 +605,55 @@ class ImageGenerationHandler(FileSystemEventHandler): resolved = os.path.abspath(image_path) return os.path.normcase(resolved) - def _prune_awaiting(self) -> None: - """Drop expired and surplus entries (``_awaiting_lock`` held).""" + def _prune_awaiting(self, store=None) -> List[Tuple[str, bool]]: + """Drop expired and surplus entries of ``store`` (the keyless set by + default; ``_awaiting_lock`` held). Returns the dropped foreign files + as ``(path, evicted)`` for ``_log_dropped`` after the lock.""" + store = self._awaiting if store is None else store now = time.monotonic() - for key in [k for k, (_p, due) in self._awaiting.items() if due <= now]: - del self._awaiting[key] - while len(self._awaiting) > MAX_AWAITING_HISTORY: - self._awaiting.popitem(last=False) + expired = [k for k, (_p, due) in store.items() if due <= now] + dropped = [(store.pop(key)[0], False) for key in expired] + while len(store) > MAX_AWAITING_HISTORY: + dropped.append((store.popitem(last=False)[1][0], True)) + return dropped if store is self._awaiting_foreign else [] - def _await_history(self, image_path: str) -> None: + def _log_dropped(self, dropped: List[Tuple[str, bool]]) -> None: + """Log foreign files that stopped waiting (outside ``_awaiting_lock``).""" + for image_path, evicted in dropped: + name = os.path.basename(image_path) + if evicted: + self.logger.info( + f"Too many files wait for their run; {name} left unlinked" + ) + else: + self.logger.info(f"No recorded run for {name}; left unlinked") + + def _await_history(self, image_path: str, store=None) -> None: """Re-check ``image_path`` once, history only, when its run is sealed. - A closed handler re-checks nothing. + ``store`` is the waiting set (``_awaiting_foreign`` for a foreign + output). A closed handler re-checks nothing. """ if self._closed: return if not self._sealed_subscribed: subscribe_outputs_sealed(self._on_outputs_sealed) self._sealed_subscribed = True + store = self._awaiting if store is None else store key = self._awaiting_key(image_path) with self._awaiting_lock: - self._awaiting.pop(key, None) - self._awaiting[key] = ( - image_path, - time.monotonic() + AWAIT_HISTORY_SECONDS, - ) - self._prune_awaiting() + store.pop(key, None) + store[key] = (image_path, time.monotonic() + AWAIT_HISTORY_SECONDS) + dropped = self._prune_awaiting(store) + self._log_dropped(dropped) - def _stop_awaiting(self, image_path: str) -> None: + def _await_foreign(self, image_path: str) -> None: + self._await_history(image_path, self._awaiting_foreign) + + def _stop_awaiting(self, image_path: str, store=None) -> None: + store = self._awaiting if store is None else store with self._awaiting_lock: - self._awaiting.pop(self._awaiting_key(image_path), None) + store.pop(self._awaiting_key(image_path), None) def awaiting_history(self) -> List[str]: """Normalized paths still waiting for their run's recorded outputs.""" @@ -626,6 +661,14 @@ class ImageGenerationHandler(FileSystemEventHandler): self._prune_awaiting() return list(self._awaiting) + def awaiting_foreign(self) -> List[str]: + """Normalized paths of foreign outputs waiting for their run (#247).""" + with self._awaiting_lock: + dropped = self._prune_awaiting(self._awaiting_foreign) + waiting = list(self._awaiting_foreign) + self._log_dropped(dropped) + return waiting + def _on_outputs_sealed(self, run_id: str, references: List[dict]) -> None: """A run's outputs were recorded: re-check its waiting images once. @@ -637,6 +680,7 @@ class ImageGenerationHandler(FileSystemEventHandler): due = [] with self._awaiting_lock: self._prune_awaiting() + dropped = self._prune_awaiting(self._awaiting_foreign) for reference in references: if reference.get("folder_type") != "output": continue @@ -647,39 +691,53 @@ class ImageGenerationHandler(FileSystemEventHandler): ) waiting = self._awaiting.pop(key, None) if waiting is not None: - due.append(waiting[0]) - for image_path in due: - self._schedule_recheck(image_path) + due.append((waiting[0], None)) + waiting = self._awaiting_foreign.pop(key, None) + if waiting is not None: + due.append((waiting[0], dict(reference))) + self._log_dropped(dropped) + for image_path, reference in due: + self._schedule_recheck(image_path, reference) - def _schedule_recheck(self, image_path: str) -> None: - """Schedule the history-only re-check, or keep it due for later. + @staticmethod + def _recheck_snapshot(reference): + return HISTORY_ONLY if reference is None else ForeignRecheck(reference) + + def _schedule_recheck(self, image_path: str, reference=None) -> None: + """Schedule the history-only re-check (with the sealed ``reference``: + a foreign output's), or keep it due for later. A closed handler drops it. """ if self._closed: return - if self.schedule(image_path, HISTORY_ONLY): + if self.schedule(image_path, self._recheck_snapshot(reference)): return with self._awaiting_lock: # ``close`` sets the flag before ``shutdown`` clears this set, so # a close that raced the refusal above is seen here. if self._closed: return - self._history_due.pop(image_path, None) - self._history_due[image_path] = None - while len(self._history_due) > MAX_AWAITING_HISTORY: - self._history_due.popitem(last=False) + due = self._history_due if reference is None else self._foreign_due + due.pop(image_path, None) + due[image_path] = reference + evicted = [] + while len(due) > MAX_AWAITING_HISTORY: + evicted.append(due.popitem(last=False)[0]) + if reference is not None: + self._log_dropped([(path, True) for path in evicted]) def _drain_history_due(self) -> None: """Schedule due re-checks now that a pending slot or path is free.""" with self._awaiting_lock: - if not self._history_due: - return - due = list(self._history_due) - for image_path in due: - if self.schedule(image_path, HISTORY_ONLY): + due = [(self._history_due, p, None) for p in self._history_due] + due += [(self._foreign_due, *item) for item in self._foreign_due.items()] + for store, image_path, reference in due: + if self.schedule(image_path, self._recheck_snapshot(reference)): with self._awaiting_lock: - self._history_due.pop(image_path, None) + # A newer seal may have replaced it meanwhile: keep that one + if image_path in store and store[image_path] is reference: + del store[image_path] def _recheck_history(self, image_path: str) -> None: """The one history-only re-check of a keyless image (#342).""" @@ -703,6 +761,108 @@ class ImageGenerationHandler(FileSystemEventHandler): metadata or {"file_info": self.get_basic_file_info(image_path)}, ) + def _ingest_foreign(self, image_path: str, metadata) -> None: + """A foreign output at settle: store its certain prompt, or wait for + its run's recorded outputs when only history can decide (#247).""" + name = os.path.basename(image_path) + if not save_foreign_runs(): + self.logger.info( + f"No PromptManager prompt in workflow, skipping image: {image_path}" + ) + return + in_output = in_output_folder(image_path, host_output_directory()) + decision = decide_foreign( + image_path, metadata, in_output=in_output, lookup=None + ) + if decision.kind == HISTORY: + decision = self._settle_lookup(image_path, metadata) + if decision is None: + self.logger.debug(f"{name} waits for its run to end") + return + self._stop_awaiting(image_path, self._awaiting_foreign) + if decision.kind in NOT_SAVED_KINDS: + self._log_not_saved(image_path, decision.kind) + elif decision.kind in CERTAIN: + self._store_foreign(image_path, decision) + else: + self.logger.info(f"No certain prompt for {name}; left unlinked") + + def _log_not_saved(self, image_path: str, kind: str = NOT_SAVED) -> None: + """The log line of a file whose run's prompt does not save (#90).""" + if kind == NOT_SAVED_WIRED: + reason = "is wired from a node the image cannot show" + else: + reason = "is off" + self.logger.info( + f"Save to library {reason} for this run's prompt, skipping image: " + f"{image_path}" + ) + + def _settle_lookup(self, image_path: str, metadata): + """The settle-time history lookup of a file only history can decide. + + Registered first, so a run sealed before this lookup is not missed. + Returns None to keep waiting: no record yet, and the answer is not + the inspector's or the recorded saver's (the legacy floor waits for + the re-check). With a record, any decision ends the wait. A + ``not_saved`` here is the undecided step's (an early one never asks + history, so it never reaches this lookup): a recorded saver may still + resolve another branch, so without a record it waits too (#90). + """ + self._await_foreign(image_path) + found = [] + lookup = history_lookup(self.db_manager, image_path) + + def recorded(): + found.append(lookup()) + return found[-1] + + decision = decide_foreign(image_path, metadata, in_output=True, lookup=recorded) + if any(record is not None for record in found): + return decision + if decision.kind in CERTAIN and decision.source in ANCHORED_SOURCES: + return decision + return None + + def _recheck_foreign(self, image_path: str, reference) -> None: + """The one re-check of a foreign output after its run ended (#247).""" + name = os.path.basename(image_path) + if not os.path.exists(image_path): + return + if not save_foreign_runs(): + self.logger.info(f"Saving prompts from other workflows is off: {name}") + return + try: + metadata = self.metadata_extractor.extract_metadata(image_path) + except Exception as error: + self.logger.warning(f"Metadata of {name} could not be read: {error}") + return + lookup = history_lookup(self.db_manager, image_path, reference) + decision = decide_foreign(image_path, metadata, in_output=True, lookup=lookup) + if decision.kind in NOT_SAVED_KINDS: + self._log_not_saved(image_path, decision.kind) + return + if decision.kind in CERTAIN: + self._store_foreign(image_path, decision) + return + self.logger.info( + f"No certain prompt for {name} after its run ended; left unlinked" + ) + + def _store_foreign(self, image_path: str, decision) -> None: + name = os.path.basename(image_path) + if not save_foreign_runs(): # switched off while this file was decided + self.logger.info(f"Saving prompts from other workflows is off: {name}") + return + try: + status, prompt_id = store_foreign(self.db_manager, image_path, decision) + except Exception as error: + self.logger.error(f"Failed to store the prompt of {name}: {error}") + return + self.logger.info( + f"Foreign run: {status} prompt {prompt_id} for {name} ({decision.source})" + ) + def _history_prompt(self, image_path: str) -> Optional[int]: """The stored prompt recorded history attributes to this file, if one.""" try: diff --git a/web/js/workspace/system-sources.mjs b/web/js/workspace/system-sources.mjs index 0433c864..6afb437e 100644 --- a/web/js/workspace/system-sources.mjs +++ b/web/js/workspace/system-sources.mjs @@ -14,6 +14,9 @@ * it to the host draft of `gallery_root_paths`; Remove takes one out. Both wait for the one Save row. * - "Watch for new outputs" (`monitoring_enabled`) and "Worker threads" (`worker_threads`, a range up to the server's * `cpu_count`, D113) are host drafts too. "Enrich existing metadata" reviews a scan of every source. + * - D158 (#247): "Save prompts from other workflows" (`save_foreign_runs`), a host draft below "Watch for new outputs", + * applied by the server at once (the monitor reads it for every file); disabled, saying why, while monitoring is off, + * unless the running monitor still watches until a restart (D128 `stopping`): then it can still be set. * - "Thumbnail generation → Review options" opens "Thumbnail maintenance" (thumbnail-review.mjs, S5 Task 14), which * starts the thumbnail Activity job. * @@ -63,6 +66,21 @@ export const LOAD_FAILED_TEXT = 'Sources could not be loaded. Your saved data ha export const DUPLICATE_TEXT = 'This folder is already a source.'; export const VALIDATE_FAILED_TEXT = 'The folder could not be checked. Try again.'; export const REFRESH_FAILED_TEXT = 'Files could not be refreshed. Try again.'; +/** D158 (#247): the row the concept does not draw, below "Watch for new outputs". */ +export const FOREIGN_RUNS = Object.freeze({ id: 'foreign-runs', key: 'save_foreign_runs', label: 'Save prompts from other workflows', + description: 'New outputs of workflows without a PromptManager node add their prompt to the library when it is certain.', + off: 'Applies while Watch for new outputs is on.' }); + +/** + * D158: the row for the drafted `save_foreign_runs` (`value`) and `monitoring_enabled` (`monitoring`): disabled until + * read, and while monitoring is off, when `note` says why (the value is kept and saved as it is). `stopping`: a source + * is still watched until the restart (D128), so the value can still be set; the note stays. + */ +export function foreignRunsState({ value, monitoring, stopping = false }) { + if (value === undefined) return { disabled: true, checked: null, note: null }; + const off = monitoring === false; + return { disabled: off && !stopping, checked: value === true, note: off ? FOREIGN_RUNS.off : null }; +} // Labels of the sources added in this page session (validate-source answers them), for the rows not saved yet. const addedLabels = new Map(); @@ -267,13 +285,16 @@ export function mountSources(container, { api, activity, scope, drafts, notify = const monitoring = settingRow({ id: 'monitoring', label: 'Watch for new outputs', description: 'New files enter the same library. Existing scans do not count as new runs.', control: { type: 'toggle' }, scope, onChange: value => drafts.set('host', 'monitoring_enabled', value) }); + const foreign = settingRow({ id: FOREIGN_RUNS.id, label: FOREIGN_RUNS.label, description: FOREIGN_RUNS.description, + control: { type: 'toggle' }, scope, onChange: value => drafts.set('host', FOREIGN_RUNS.key, value) }); const workers = settingRow({ id: 'workers', label: 'Worker threads', description: 'Balance background scans and thumbnail work against generation.', control: { type: 'range', value: 4 }, scope, onChange: value => drafts.set('host', 'worker_threads', value) }); - const hostRows = [[monitoring, 'monitoring', 'monitoring_enabled'], [workers, 'workers', 'worker_threads']] + const hostRows = [[monitoring, 'monitoring', 'monitoring_enabled'], [foreign, FOREIGN_RUNS.id, FOREIGN_RUNS.key], + [workers, 'workers', 'worker_threads']] .map(([row, id, key]) => ({ input: controlOf(row, id), key, text: row.children[0], note: null })); for (const row of hostRows) row.input.dataset.key = row.key; - surface.append(monitoring, workers, + surface.append(monitoring, foreign, workers, operationRow({ scope, title: 'Enrich existing metadata', description: 'Resolve both prompts, deduplicate recipes, preserve uncertain links.', label: 'Review scan', onClick: scanAll }), operationRow({ scope, title: 'Thumbnail generation', description: 'Generate missing previews; rebuilding all is a separate explicit option.', @@ -348,6 +369,10 @@ export function mountSources(container, { api, activity, scope, drafts, notify = for (const row of hostRows) { const value = drafts.get('host', row.key); row.input.disabled = value === undefined; + if (row.key === FOREIGN_RUNS.key) { + paintForeign(row, value, unavailable); + continue; + } if (row.key === 'worker_threads') { // D113: the range ends at the server's core count (the concept's fixed 16). const cpus = drafts.get('host', 'cpu_count'); @@ -360,6 +385,24 @@ export function mountSources(container, { api, activity, scope, drafts, notify = } } + // D158: disabled while monitoring is drafted off, with the reason under the description. + function paintForeign(row, value, unavailable) { + const stopping = sources?.effective?.some(source => source.watch === 'stopping') === true; + const state = foreignRunsState({ value, monitoring: drafts.get('host', 'monitoring_enabled'), stopping }); + row.input.disabled = state.disabled; + if (state.checked !== null) row.input.checked = state.checked; + const note = unavailable && value === undefined ? HOST_UNAVAILABLE : state.note; + if (note && row.note?.textContent !== note) { + row.note?.remove(); + row.note = el('p', note, 'small'); + row.note.id = `${FOREIGN_RUNS.id}-note`; + row.text.append(row.note); + } else if (!note && row.note) { row.note.remove(); row.note = null; } + // The note is the switch's description, so a screen reader says why it is disabled. + if (row.note) row.input.setAttribute('aria-describedby', row.note.id); + else row.input.removeAttribute('aria-describedby'); + } + // The sources changed since this page read them (a refused per-source scan): read them and the saved values again. function reread() { load(); @@ -378,6 +421,7 @@ export function mountSources(container, { api, activity, scope, drafts, notify = if (answer === STALE || scope.signal.aborted) return; sources = answer?.sources ?? null; paintList(); + paintControls(); } // A save (or a reread after a conflict) changed the saved sources or monitoring: read their state again. The first From b5ca2eda940470e3c7b7b36a0c0fed4b53990ee8 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Thu, 29 Oct 2026 12:00:36 +0000 Subject: [PATCH 3/7] feat(tools): reviewed repair of the foreign dataset links (#247, PR 3) (#468) * feat(inspector): use recorded run history when the graph alone can't name the prompt (#386) When a file's own graph leaves the prompt open, both inspect routes ask recorded run history about the file (one run reported it and it was written during that run). The save node that run reported it from anchors the walk like a file-name match, so dataset branches that share one saver prefix show their own prompt, labelled "Recorded run". Still open at one sampler whose PromptManager text was recorded at execution (a language model's output), the recorded text shows as "Execution recorded". Results the file resolves alone never change; a failed lookup judges the file alone. * fix(inspector): recorded texts follow history's saver; conflicts anchor nothing (#386) Review follow-ups: the texts PromptManager recorded are looked up at the sampler history's saver leads to, never at a sampler the unanchored walk named; a file name naming a different saver than history anchors nothing; outputs and texts are read in one transaction; "Recorded run" ranks below every workflow-inferred source; unexpected lookup errors log at error level. * fix(history): look up recorded runs by history's own spelling of the file (#386) The history route inspects a case-normalized target on Windows, while history matches file names exactly, so the occurrence's reference names the file. * refactor(inspector): share the inspector's resolve as inspection_resolution inspect_prompt_evidence now calls inspection_resolution, which returns the resolution to use, the anchored result and whether the recorded saver chose it. Ingestion can call the same helper for files the default profile cannot decide (#247). * feat(ingestion): one per-file prompt decision for the scan and the monitor utils/file_prompt.py holds decide_file_prompt, the ladder the output scan runs today plus the inspector's rule (inspection profile, then the recorded saver) before the legacy parser, with a SCAN and a MONITOR policy. Junk text (blank, numbers, LoRA tags only, model file names) is never stored. graph_prompt_text and parse_comfyui_prompt move here; the parser accepts parsed metadata (#247). * feat(ingestion): store a file's prompt in one transaction, one prompt per file database/scanned_prompts.py links a file to the prompt of one text, creating it when new, inside one BEGIN IMMEDIATE transaction: a file linked to another prompt is a conflict, a failed link leaves no prompt behind and two writers store one prompt. pair_ingestion gains file_link_rows, link_conflict and Marks/mark_created; ingest_resolved_pair takes optional marks for the components it creates. recorded_runs and named_prompt in run_attribution become public (#247). * feat(scan): decide each scanned file with the shared ladder (#247) _ingest_scanned_file calls decide_file_prompt (scan policy) and stores the result: pairs through ingest_resolved_pair and positives through store_scanned_prompt, both with the scan marks. Files the default profile leaves undecided now get the inspector's resolve, with the saver recorded history names, before the legacy parser, so each branch of a shared-prefix dataset run links its own prompt. Junk text is never stored, a file linked to another prompt is a conflict instead of a second link, and a failed link leaves no prompt behind. _BatchHistory reads the stored prompt and the recorded saver from one recorded_runs pass. _parse_comfyui_prompt calls the moved parse_comfyui_prompt. TESTING.md documents the decision. * fix(ingestion): keep the file-name saver anchor out of the ladder (#390) The #390 ruling queues a file that only its saver's file name resolves for Needs review, never linking it. inspection_resolution gains name_anchor; the ladder passes False, so such a file falls to the legacy floor in the scan (and to Needs review) and is never stored by the monitor, while the recorded saver still decides. The inspector keeps the name anchor. * fix(ingestion): no joined PromptManager text from the legacy floor; junk keeps its step's status The legacy parser returns prepend + text + append, so its answer is refused when a PromptManager-family node has a prepend or append part (#343). Junk found at the undecided-graph step keeps that step's unlinked status and review, so an ambiguous file is still queued for Needs review. The monitor policy's ambiguous file no longer asks for the recorded run, and the file size may be read lazily. * fix(ingestion): keep the legacy answer that is a PromptManager node's own text The joined-text refusal also refused the #170 parity files whose PromptManager prepend is a link the browser parser does not follow: its answer is then the node's literal text, which is what the node stores. Refuse only an answer that is not such a node's own text. * fix(scan): one prompt per file for recorded-run links; writes survive a library move The recorded run's stored prompt now links in one transaction with the file's link rows, so a file linked to another prompt is a conflict instead of a second link. The scan's stores run through survive_library_switch. A single file asks history once (a one-file _BatchHistory), the scan passes in_output=False and reads the file size only for files with prompt metadata. A conflict row reads "Kept: already linked to a different prompt". TESTING.md records the rules. * feat(tools): reviewed repair of the foreign dataset links (#247) tools/repair_foreign_links.py judges every link row of a file that only foreign runs reported with the shared ladder (scan policy, recorded saver). The dry run judges a copy, writes a JSON plan and a Markdown list, and rehearses the apply on that copy to audit the prompts that would lose every link (deleting them is off unless chosen). --apply refuses while ComfyUI runs, takes a backup, re-judges every row inside one BEGIN IMMEDIATE transaction, creates the targets with the scan's marks, relinks and drops pair links, keeps UNIQUE(prompt_id, file_path), and rolls back with exit 2 on any drift. * fix(tools): the repair list keeps whole file paths and shows each target hash * fix(tools): harden the #247 repair apply after review - refuse while any other process holds the library, its -wal or -shm open (tools/comfyui_stopped.py), also in a rehearsal; without /proc only with --i-stopped-comfyui; require main.py in --comfyui-dir; check again just before BEGIN IMMEDIATE - the plan records its library (normcase realpath) and max row id; apply refuses on another library - refuse when opening the library changes its schema (run from a checkout at the live code version) - verify the backup (integrity_check and counts) and never overwrite one - target_text is a drift key and must match its hash; keep compares the linked text's hash, not the stored column - the dry run rehearses the orphan deletes when --delete-orphans is chosen - a .md plan path is refused; the list is <stem>.list.md - any error during apply rolls back, names the backup, exits 2 - relpath on another drive falls back to the full path * fix(ingestion): match a file's link rows under its resolved spelling too The scan stores realpath spellings and the monitor the watcher's raw path, so a file reached through a symlinked or junction output folder could get a second generated_images row. file_link_rows now queries the path and its realpath (each normalized and raw) and dedupes by row id. TESTING.md notes the reverse order is matched only when the raw spelling is the realpath. * fix(ingestion): a switched-off PromptManager node behind a pass-through stores nothing (#90) The #90 graph walk follows only prompt-like inputs, so a PromptManagerText with Save to library off reaching the encoder through another input name (easy showAnything.anything, a custom node's value) was invisible to it, and the inspector's resolve stored its text. The ladder now checks the positive every step names: a resolved or inspected positive whose path passes a node that is off, or a legacy answer that is such a node's own text, makes the file not_saved. * fix(ingestion): any switched-off node upstream of a positive stores nothing (#90) The ladder walks every input upstream of each sampler or guider positive (not negative, model, clip, latent or image inputs) for PromptManager nodes whose Save to library is off or wired from an unreadable node, so the sampler-less step, an off node with a prepend part and a PromptManagerPair behind a pass-through no longer store its text. The legacy-floor guard reads a node's resolved text. Every guard reports not_saved for a switch that is off and not_saved_wired for one it cannot read. * fix(ingestion): the switch walk skips negative inputs by #90's own rule The upstream walk treated only names containing "negative" as negative, so an off node on a neg, neg_text or text_neg input blocked an enabled positive. prompt_graph gains is_negative_input (the _NEGATIVE_KEY rule, also used by follows_input) and the walk uses it. The guards accept a node with null inputs and a side whose provenance is not an object. * fix(ingestion): the switch walk also skips any input named with "negative" #90's rule matches only whole underscore-separated words, so negative2, negativeprompt and NegativePrompt counted as positive inputs and an off node wired only to one of them blocked the file. The walk skips those too; follows_input is unchanged. * test(tools): the #247 repair never relinks a switched-off node's text (#90) The plan decides every file through decide_file_prompt, so a not_saved or not_saved_wired file is a skip. Cover an off node behind a pass-through, an off node in the recorded saver's branch (literal and wired switch), an off node on another branch, and an off node wired only to the negative (still repaired). --- tests/test_repair_foreign_links.py | 951 +++++++++++++++++++++++++++++ tools/comfyui_stopped.py | 81 +++ tools/repair_foreign_links.py | 776 +++++++++++++++++++++++ 3 files changed, 1808 insertions(+) create mode 100644 tests/test_repair_foreign_links.py create mode 100644 tools/comfyui_stopped.py create mode 100644 tools/repair_foreign_links.py diff --git a/tests/test_repair_foreign_links.py b/tests/test_repair_foreign_links.py new file mode 100644 index 00000000..61a81910 --- /dev/null +++ b/tests/test_repair_foreign_links.py @@ -0,0 +1,951 @@ +"""tools/repair_foreign_links.py: the reviewed repair of the #247 dataset links. + +The fixture is a library whose foreign (no PromptManager node) dataset run +wrote shared-prefix files that the old scan linked to the wrong branch text. +Recorded history names each file's saver, so the ladder (``recorded-run``) +names the right branch. +""" + +import contextlib +import hashlib +import io +import json +import os +import shutil +import sqlite3 +import tempfile +import unittest +import unittest.mock +from pathlib import Path + +from PIL import Image +from PIL.PngImagePlugin import PngInfo + +try: + from tests.open_handles import assert_closed +except ImportError: # discovered with tests/ as the top-level directory + from open_handles import assert_closed + +from database.models import PromptModel +from database.operations import PromptDatabase +from tests import saver_anchor_graphs as shapes +from tests.test_run_attribution import RecordedRuns +from tools import repair_foreign_links as tool +from utils.file_prompt import Decision +from utils.hashing import generate_prompt_hash + +TEXTS = shapes.BRANCH_TEXTS +NAME = "IMG_GEN_{:05}_.png" + + +def png(path, graph): + info = PngInfo() + info.add_text("prompt", json.dumps(graph)) + Image.new("RGB", (4, 4)).save(path, pnginfo=info) + + +def sha(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def file_sha(path): + return hashlib.sha256(Path(path).read_bytes()).hexdigest() + + +def dump(path): + """The database's logical content (independent of WAL checkpoints).""" + conn = sqlite3.connect(path) + try: + return list(conn.iterdump()) + finally: + conn.close() + + +def not_running(_comfyui_dir, _port): + return [] + + +def no_handles(_database): + return [] + + +class RepairFixture(RecordedRuns, unittest.TestCase): + """Rows: A relink to a new prompt, B relink to a stored prompt (with a + pair link), C keep, D file gone, E two savers (skip), F the UNIQUE case + (a wrong row beside a correct row), H relink from a rated prompt, G a + PromptManager run (not a candidate).""" + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.addCleanup(assert_closed, self, self.tmp) + self.addCleanup(PromptModel.reset_schema_cache) + self.output = os.path.join(self.tmp, "output") + os.makedirs(os.path.join(self.output, "dataset")) + self.db_path = os.path.join(self.tmp, "prompts.db") + self.plan_path = os.path.join(self.tmp, "plan.json") + self.backups = os.path.join(self.tmp, "backups") + os.makedirs(self.backups) + self.comfy = os.path.join(self.tmp, "comfy") + os.makedirs(self.comfy) + Path(self.comfy, "main.py").write_text("# ComfyUI\n", encoding="utf-8") + self.files = {} + for key, number in zip("ABCDEFHG", range(1, 9)): + path = os.path.join(self.output, "dataset", NAME.format(number)) + png(path, shapes.shared_prefix()) + self.files[key] = path + db = PromptDatabase(self.db_path) + try: + self.build(db) + finally: + db.close_all() + os.remove(self.files["D"]) + + def build(self, db): + save = lambda text, **kw: db.save_prompt( # noqa: E731 + text=text, prompt_hash=generate_prompt_hash(text), **kw + ) + self.p0 = save(TEXTS[0], category="scanned", tags=["auto-scanned"]) + self.p2 = save(TEXTS[2], category="scanned", tags=["auto-scanned"]) + self.orphan = save( + "an old guess", + category="scanned", + tags=["auto-scanned"], + notes="Auto-scanned from IMG_GEN_00001_.png", + ) + self.rated = save("a rated guess", rating=5) + link = lambda prompt, key: db.link_image_to_prompt( # noqa: E731 + prompt, self.files[key], metadata={} + ) + self.rows = { + "A": link(self.orphan, "A"), + "B": link(self.p0, "B"), + "C": link(self.p0, "C"), + "D": link(self.p0, "D"), + "E": link(self.p0, "E"), + "F": link(self.p0, "F"), + "F_ok": link(self.p2, "F"), + "H": link(self.rated, "H"), + "G": link(self.p0, "G"), + } + conn = db.model.get_connection() + with conn: + pair = conn.execute( + "INSERT INTO prompt_pairs(positive_prompt_id) VALUES(?)", (self.p0,) + ).lastrowid + conn.execute( + "INSERT INTO image_prompt_pairs(image_id, pair_id, provenance) " + "VALUES(?,?,'{}')", + (self.rows["B"], pair), + ) + name = lambda key: ("dataset", os.path.basename(self.files[key])) # noqa + # Savers of the shared-prefix graph: "13", "23", "33", "43" (branches 1-4) + self.record_run( + db, + self.output, + [name(k) for k in "CDEAEBFH"], + [], + nodes=["13", "13", "13", "23", "23", "33", "33", "43"], + ) + self.record_run(db, self.output, [name("G")], [TEXTS[0]], item=1, nodes=["23"]) + + def dry_run(self, *args): + out = io.StringIO() + code = tool.main( + [ + self.db_path, + "--output-dir", + self.output, + "--plan", + self.plan_path, + *args, + ], + out=out, + ) + return code, out.getvalue() + + def apply(self, *args, probe=not_running, handles=no_handles): + out = io.StringIO() + code = tool.main( + [ + self.db_path, + "--apply", + self.plan_path, + "--comfyui-dir", + os.path.join(self.tmp, "comfy"), + "--backup-dir", + self.backups, + *args, + ], + out=out, + probe=probe, + handles=handles, + ) + return code, out.getvalue() + + def plan(self): + with open(self.plan_path, encoding="utf-8") as stream: + return json.load(stream) + + def write_plan(self, plan): + with open(self.plan_path, "w", encoding="utf-8") as stream: + json.dump(plan, stream) + + def query(self, sql, args=()): + conn = sqlite3.connect(self.db_path) + try: + return conn.execute(sql, args).fetchall() + finally: + conn.close() + + def query_write(self, sql, args): + conn = sqlite3.connect(self.db_path) + try: + with conn: + conn.execute(sql, args) + finally: + conn.close() + + def links(self): + return dict(self.query("SELECT id, prompt_id FROM generated_images")) + + +class DryRunTests(RepairFixture): + def test_plan_names_relink_keep_and_skip_rows(self): + code, out = self.dry_run() + self.assertEqual(code, 0, out) + items = {item["row_id"]: item for item in self.plan()["rows"]} + rows = self.rows + self.assertNotIn(rows["G"], items, "a PromptManager run is not foreign") + actions = {key: items[rows[key]]["action"] for key in rows if key != "G"} + self.assertEqual( + actions, + { + "A": "relink", + "B": "relink", + "C": "keep", + "D": "skip", + "E": "skip", + "F": "relink", + "F_ok": "keep", + "H": "relink", + }, + ) + a = items[rows["A"]] + self.assertEqual(a["current_prompt_id"], self.orphan) + self.assertEqual(a["current_text_sha256"], sha("an old guess")) + self.assertEqual(a["target_text"], TEXTS[1]) + self.assertEqual(a["target_hash"], generate_prompt_hash(TEXTS[1])) + self.assertEqual(a["target_prompt_id"], "new") + self.assertEqual(a["file"], self.files["A"]) + self.assertEqual(items[rows["B"]]["target_prompt_id"], self.p2) + self.assertEqual(items[rows["H"]]["target_prompt_id"], "new") + self.assertEqual(items[rows["D"]]["reason"], "file gone") + self.assertEqual( + items[rows["E"]]["reason"], "not decided by the recorded saver" + ) + self.assertIn("relink: 4", out) + self.assertIn("keep: 2", out) + self.assertIn("skip: 2", out) + + def test_orphan_audit_lists_the_prompts_losing_every_link(self): + self.dry_run() + plan = self.plan() + orphans = {o["id"]: o for o in plan["orphans"]} + self.assertEqual(set(orphans), {self.orphan, self.rated}) + self.assertEqual(orphans[self.orphan]["blockers"], []) + self.assertTrue(orphans[self.orphan]["safe"]) + self.assertEqual(orphans[self.rated]["blockers"], ["usage or rating"]) + self.assertFalse(orphans[self.rated]["safe"]) + self.assertFalse(plan["delete_orphans"], "off by default") + self.dry_run("--delete-orphans") + self.assertTrue(self.plan()["delete_orphans"]) + + def test_dry_run_writes_nothing_and_writes_the_markdown_list(self): + before = file_sha(self.db_path) + listing = sorted(os.listdir(self.tmp)) + code, out = self.dry_run() + self.assertEqual(code, 0) + self.assertEqual(file_sha(self.db_path), before) + self.assertIn("Dry run: nothing was changed.", out) + # A read-only reader of a WAL library may leave its empty side files + sidecars = {"prompts.db-shm", "prompts.db-wal"} + self.assertEqual( + sorted(set(os.listdir(self.tmp)) - sidecars), + sorted(set(listing + ["plan.json", "plan.list.md"]) - sidecars), + ) + wal = Path(self.tmp, "prompts.db-wal") + self.assertEqual(wal.stat().st_size if wal.exists() else 0, 0) + markdown = Path(self.tmp, "plan.list.md").read_text(encoding="utf-8") + self.assertIn(f"| {self.rows['A']} |", markdown) + long_path = os.path.join("dataset", NAME.format(1)) + self.assertIn(f"| {long_path} |", markdown, "file paths are never cut") + self.assertIn(generate_prompt_hash(TEXTS[1])[:12], markdown) + self.assertIn("new", markdown) + self.assertIn("would lose every link", markdown) + + +class DryRunReviewTests(RepairFixture): + def test_plan_records_its_library_and_fingerprint(self): + self.dry_run() + plan = self.plan() + self.assertEqual( + plan["source_db"], os.path.normcase(os.path.realpath(self.db_path)) + ) + self.assertEqual(plan["max_row_id"], max(self.links())) + + def test_a_markdown_plan_path_is_refused(self): + out = io.StringIO() + with ( + self.assertRaises(SystemExit), + contextlib.redirect_stderr(io.StringIO()), + ): + tool.main( + [ + self.db_path, + "--output-dir", + self.output, + "--plan", + os.path.join(self.tmp, "plan.md"), + ], + out=out, + ) + self.assertFalse(os.path.exists(os.path.join(self.tmp, "plan.md"))) + + def test_keep_compares_the_linked_text_not_the_stored_hash(self): + self.query_write("UPDATE prompts SET hash='stale' WHERE id=?", (self.p0,)) + self.dry_run() + items = {item["row_id"]: item for item in self.plan()["rows"]} + self.assertEqual(items[self.rows["C"]]["action"], "keep") + + def test_delete_orphans_is_rehearsed_with_the_deletes(self): + code, out = self.dry_run("--delete-orphans") + self.assertEqual(code, 0, out) + simulated = self.plan()["simulated"] + self.assertEqual(simulated["result"]["orphans_deleted"], 1) + self.assertEqual( + simulated["after"]["prompts"], simulated["before"]["prompts"] + 2 - 1 + ) + + def test_a_file_on_another_drive_is_listed_by_its_full_path(self): + self.dry_run() + plan = self.plan() + with unittest.mock.patch.object( + tool.os.path, "relpath", side_effect=ValueError("other drive") + ): + listing = tool.markdown(plan, {}) + self.assertIn(self.files["A"], listing) + + +class ApplyTests(RepairFixture): + def setUp(self): + super().setUp() + code, out = self.dry_run() + self.assertEqual(code, 0, out) + + def test_apply_relinks_creates_targets_and_drops_pair_links(self): + code, out = self.apply() + self.assertEqual(code, 0, out) + links = self.links() + new = dict(self.query("SELECT hash, id FROM prompts")) + target_a = new[generate_prompt_hash(TEXTS[1])] + target_h = new[generate_prompt_hash(TEXTS[3])] + self.assertEqual(links[self.rows["A"]], target_a) + self.assertEqual(links[self.rows["B"]], self.p2) + self.assertEqual(links[self.rows["H"]], target_h) + self.assertEqual(links[self.rows["C"]], self.p0, "keep") + self.assertEqual(links[self.rows["E"]], self.p0, "skip") + self.assertEqual(links[self.rows["G"]], self.p0, "not foreign") + self.assertEqual( + self.query( + "SELECT COUNT(*) FROM image_prompt_pairs WHERE image_id=?", + (self.rows["B"],), + ), + [(0,)], + ) + category, notes = self.query( + "SELECT category, notes FROM prompts WHERE id=?", (target_a,) + )[0] + self.assertEqual(category, "scanned") + self.assertEqual( + notes, f"Auto-scanned from {os.path.basename(self.files['A'])}" + ) + tags = self.query( + "SELECT t.name FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id " + "WHERE pt.prompt_id=?", + (target_a,), + ) + self.assertEqual(tags, [("auto-scanned",)]) + self.assertIn("integrity_check", out) + self.assertIn("foreign_key_check", out) + self.assertRegex(out, r"integrity_check\s+ok\s+ok") + # Orphans are kept unless the plan chose to delete them + self.assertTrue(self.query("SELECT 1 FROM prompts WHERE id=?", (self.orphan,))) + + def test_unique_case_deletes_the_wrong_row(self): + code, out = self.apply() + self.assertEqual(code, 0, out) + links = self.links() + self.assertNotIn(self.rows["F"], links) + self.assertEqual(links[self.rows["F_ok"]], self.p2) + rows = self.query( + "SELECT prompt_id FROM generated_images WHERE image_path=?", + (self.files["F"],), + ) + self.assertEqual(rows, [(self.p2,)]) + + def test_backup_is_taken_before_writing(self): + before = self.links() + code, out = self.apply() + self.assertEqual(code, 0, out) + names = os.listdir(self.backups) + self.assertEqual(len(names), 1) + self.assertRegex(names[0], r"^prompts\.db\.pre-247-repair_\d{8}_\d{6}$") + backup = os.path.join(self.backups, names[0]) + conn = sqlite3.connect(backup) + try: + saved = dict(conn.execute("SELECT id, prompt_id FROM generated_images")) + finally: + conn.close() + self.assertEqual(saved, before) + self.assertIn(f"Backup: {backup}", out) + + def test_backup_beside_the_database_by_default(self): + out = io.StringIO() + code = tool.main( + [ + self.db_path, + "--apply", + self.plan_path, + "--comfyui-dir", + os.path.join(self.tmp, "comfy"), + ], + out=out, + probe=not_running, + handles=no_handles, + ) + self.assertEqual(code, 0, out.getvalue()) + self.assertTrue( + any( + n.startswith("prompts.db.pre-247-repair_") for n in os.listdir(self.tmp) + ) + ) + + def test_refuses_while_comfyui_runs(self): + before = file_sha(self.db_path) + code, out = self.apply(probe=lambda _d, _p: ["process 42 (main.py)"]) + self.assertEqual(code, 2) + self.assertIn("REFUSED", out) + self.assertIn("process 42", out) + self.assertEqual(os.listdir(self.backups), [], "no backup either") + self.assertEqual(file_sha(self.db_path), before) + + def test_rehearsal_skips_the_running_check_on_a_copy_only(self): + code, out = self.apply("--rehearsal", probe=lambda _d, _p: ["port 8188"]) + self.assertEqual(code, 0, out) + self.assertIn("rehearsal", out) + # The live library of that ComfyUI folder: --rehearsal is refused + live_dir = os.path.join(self.tmp, "comfy", "user", "__promptmanager") + os.makedirs(live_dir) + live = os.path.join(live_dir, "prompts.db") + shutil.copy(self.db_path, live) + out = io.StringIO() + code = tool.main( + [ + live, + "--apply", + self.plan_path, + "--comfyui-dir", + os.path.join(self.tmp, "comfy"), + "--rehearsal", + ], + out=out, + probe=lambda _d, _p: [], + handles=no_handles, + ) + self.assertEqual(code, 2) + self.assertIn("--rehearsal", out.getvalue()) + + def test_drift_refuses_and_changes_nothing(self): + cases = { + "row moved": lambda: self.query_write( + "UPDATE generated_images SET prompt_id=? WHERE id=?", + (self.p2, self.rows["H"]), + ), + "text changed": lambda: self.query_write( + "UPDATE prompts SET text='an older guess' WHERE id=?", (self.orphan,) + ), + "file changed": lambda: png(self.files["B"], shapes.branch_bench()), + "target stored since": lambda: self.query_write( + "INSERT INTO prompts(text, hash) VALUES(?,?)", + (TEXTS[1], generate_prompt_hash(TEXTS[1])), + ), + } + pristine = os.path.join(self.tmp, "pristine.db") + shutil.copy(self.db_path, pristine) + image = Path(self.files["B"]).read_bytes() + for name, drift in cases.items(): + with self.subTest(name): + shutil.copy(pristine, self.db_path) + Path(self.files["B"]).write_bytes(image) + shutil.rmtree(self.backups) + os.makedirs(self.backups) + drift() + before = dump(self.db_path) + code, out = self.apply() + self.assertEqual(code, 2, out) + self.assertIn("REFUSED (rolled back, nothing changed)", out) + self.assertEqual(dump(self.db_path), before) + + def test_late_failure_rolls_back_every_write(self): + plan = self.plan() + plan["delete_orphans"] = True + self.write_plan(plan) + # The safe orphan gained a user tag after the dry run + self.query_write("INSERT INTO tags(name) VALUES('mine')", ()) + self.query_write( + "INSERT INTO prompt_tags(prompt_id, tag_id) " + "SELECT ?, id FROM tags WHERE name='mine'", + (self.orphan,), + ) + before = self.links() + prompts = self.query("SELECT COUNT(*) FROM prompts") + code, out = self.apply() + self.assertEqual(code, 2, out) + self.assertIn(f"prompt {self.orphan}", out) + self.assertIn("user tags", out) + self.assertEqual(self.links(), before, "the relinks were rolled back") + self.assertEqual(self.query("SELECT COUNT(*) FROM prompts"), prompts) + + def test_orphans_are_deleted_only_when_chosen_and_unreferenced(self): + plan = self.plan() + plan["delete_orphans"] = True + self.write_plan(plan) + code, out = self.apply() + self.assertEqual(code, 0, out) + self.assertFalse(self.query("SELECT 1 FROM prompts WHERE id=?", (self.orphan,))) + self.assertTrue( + self.query("SELECT 1 FROM prompts WHERE id=?", (self.rated,)), + "a prompt with a rating is never deleted", + ) + self.assertIn("orphans_deleted", out) + + def test_an_applied_plan_refuses_a_second_time(self): + self.assertEqual(self.apply()[0], 0) + before = dump(self.db_path) + code, out = self.apply() + self.assertEqual(code, 2) + self.assertIn("REFUSED", out) + self.assertEqual(dump(self.db_path), before) + + +class ApplySafetyTests(RepairFixture): + def setUp(self): + super().setUp() + code, out = self.dry_run() + self.assertEqual(code, 0, out) + + def assert_unchanged(self, before, code, out): + self.assertEqual(code, 2, out) + self.assertEqual(dump(self.db_path), before) + + def test_refuses_when_another_process_has_the_library_open(self): + before = dump(self.db_path) + held = lambda _db: ["process 7 has prompts.db-wal open"] # noqa: E731 + for args in ((), ("--rehearsal",)): + with self.subTest(args): + code, out = self.apply(*args, handles=held) + self.assert_unchanged(before, code, out) + self.assertIn("process 7", out) + self.assertEqual(os.listdir(self.backups), []) + + def test_the_running_check_runs_again_just_before_writing(self): + calls = [] + + def late(_d, _p): + calls.append(1) + return [] if len(calls) == 1 else ["something listens on port 8188"] + + before = dump(self.db_path) + code, out = self.apply(probe=late) + self.assert_unchanged(before, code, out) + self.assertEqual(len(calls), 2) + self.assertIn("rolled back", out) + self.assertIn("Backup:", out) + + def test_without_proc_only_an_explicit_stop_lets_it_through(self): + blind = lambda _db: [tool.NO_PROC] # noqa: E731 + before = dump(self.db_path) + code, out = self.apply(handles=blind) + self.assert_unchanged(before, code, out) + self.assertIn("--i-stopped-comfyui", out) + code, out = self.apply("--i-stopped-comfyui", handles=blind) + self.assertEqual(code, 0, out) + + def test_a_folder_without_main_py_is_refused(self): + os.remove(os.path.join(self.comfy, "main.py")) + before = dump(self.db_path) + code, out = self.apply() + self.assert_unchanged(before, code, out) + self.assertIn("main.py", out) + + def test_a_plan_for_another_library_is_refused(self): + other = os.path.join(self.tmp, "other.db") + shutil.copy(self.db_path, other) + out = io.StringIO() + code = tool.main( + [other, "--apply", self.plan_path, "--comfyui-dir", self.comfy], + out=out, + probe=not_running, + handles=no_handles, + ) + self.assertEqual(code, 2) + self.assertIn("made for", out.getvalue()) + + def test_opening_the_library_must_not_change_its_schema(self): + self.query_write("DROP INDEX idx_prompt_pairs_positive_only", ()) + PromptModel.reset_schema_cache() + code, out = self.apply() + self.assertEqual(code, 2, out) + self.assertIn("schema", out) + self.assertIn("live code version", out) + + def test_an_existing_backup_file_is_refused(self): + stamp = "20261028_000000" + Path(self.backups, f"prompts.db.pre-247-repair_{stamp}").write_bytes(b"x") + before = dump(self.db_path) + with unittest.mock.patch.object(tool, "_stamp", return_value=stamp): + code, out = self.apply() + self.assert_unchanged(before, code, out) + self.assertIn("already exists", out) + + def test_a_backup_that_does_not_match_is_refused(self): + real = tool.backup_database + + def short(source, target): + real(source, target) + conn = sqlite3.connect(target) + try: + with conn: + conn.execute("DELETE FROM image_prompt_pairs") + finally: + conn.close() + + before = dump(self.db_path) + with unittest.mock.patch.object(tool, "backup_database", short): + code, out = self.apply() + self.assert_unchanged(before, code, out) + self.assertIn("backup", out) + + def test_a_changed_target_text_is_refused(self): + plan = self.plan() + for item in plan["rows"]: + if item["row_id"] == self.rows["A"]: + item["target_text"] = "something else" + self.write_plan(plan) + before = dump(self.db_path) + code, out = self.apply() + self.assert_unchanged(before, code, out) + self.assertIn("target_text", out) + + def test_an_orphan_gone_or_changed_at_apply_is_refused(self): + plan = self.plan() + plan["delete_orphans"] = True + plan["orphans"][0]["text_sha256"] = "0" * 64 + self.write_plan(plan) + before = dump(self.db_path) + code, out = self.apply() + self.assert_unchanged(before, code, out) + self.assertIn("gone or text changed", out) + + def test_any_error_rolls_back_and_names_the_backup(self): + before = dump(self.db_path) + with unittest.mock.patch.object( + tool, "_relink", side_effect=RuntimeError("disk on fire") + ): + code, out = self.apply() + self.assert_unchanged(before, code, out) + self.assertIn("rolled back", out) + self.assertIn("disk on fire", out) + self.assertIn(self.backups, out) + + def test_a_failed_integrity_check_after_writing_exits_1(self): + real = tool.counts + seen = [] + + def counts(path, readonly=True): + found = real(path, readonly) + if path == self.db_path: + seen.append(path) + if len(seen) > 1: + return {**found, "integrity_check": "corrupt"} + return found + + with unittest.mock.patch.object(tool, "counts", counts): + code, out = self.apply() + self.assertEqual(code, 1, out) + self.assertIn("restore", out) + + +class BlockerTests(RepairFixture): + """Each reference that keeps a prompt from being deleted.""" + + def test_every_blocker(self): + pid = self.orphan + cases = { + "linked images": None, + "saved pairs": "INSERT INTO prompt_pairs(positive_prompt_id) VALUES(:id)", + "recorded runs": "UPDATE execution_run_contexts " + "SET positive_prompt_id=:id", + "library state": "INSERT INTO prompt_library_state(prompt_id) " + "VALUES(:id)", + "user tags": "INSERT INTO prompt_tags(prompt_id, tag_id) " + "SELECT :id, id FROM tags WHERE name='mine'", + "usage or rating": "UPDATE prompts SET run_count=3 WHERE id=:id", + "category": "UPDATE prompts SET category='portraits' WHERE id=:id", + "notes": "UPDATE prompts SET notes='mine' WHERE id=:id", + } + conn = sqlite3.connect(self.db_path) + try: + conn.execute("INSERT INTO tags(name) VALUES('mine')") + conn.commit() + for name, sql in cases.items(): + with self.subTest(name): + conn.execute("SAVEPOINT blocker") + if sql is None: + unused = {pid} + else: + conn.execute( + "DELETE FROM generated_images WHERE prompt_id=?", (pid,) + ) + conn.execute(sql, {"id": pid}) + unused = {pid} + found = tool.blockers(conn, pid, unused) + self.assertTrue(any(item.startswith(name) for item in found), found) + conn.execute("ROLLBACK TO blocker") + conn.execute("RELEASE blocker") + conn.execute("DELETE FROM generated_images WHERE prompt_id=?", (pid,)) + self.assertEqual(tool.blockers(conn, pid, {pid}), []) + self.assertEqual( + tool.blockers(conn, pid, set()), ["not unused by the app's rule"] + ) + conn.rollback() + finally: + conn.close() + + +def _off_text_node(text, save=False): + return { + "class_type": "PromptManagerText", + "inputs": {"text": text, "save_to_library": save}, + } + + +def _encode_from(source): + return { + "class_type": "CLIPTextEncode", + "inputs": {"text": source, "clip": ["1", 1]}, + } + + +def off_behind_pass_through(branch=2): + """The dataset graph with ``branch``'s text coming from a switched-off + PromptManagerText through a pass-through the #90 walk does not follow.""" + graph = shapes.shared_prefix() + graph["50"] = _off_text_node(TEXTS[branch - 1]) + graph["51"] = {"class_type": "MyNode", "inputs": {"value": ["50", 0]}} + graph[str(branch * 10)] = _encode_from(["51", 0]) + return graph + + +def off_target(branch=2, save=False): + """``branch``'s text straight from a PromptManagerText whose switch is + ``save`` (False, or a link to a node the graph cannot read).""" + graph = shapes.shared_prefix() + graph["50"] = _off_text_node(TEXTS[branch - 1], save) + graph[str(branch * 10)] = _encode_from(["50", 0]) + if isinstance(save, list): + graph["52"] = {"class_type": "SomeBoolNode", "inputs": {}} + return graph + + +def off_negative_only(): + """Every branch's negative from a switched-off PromptManagerText.""" + graph = shapes.shared_prefix() + graph["50"] = _off_text_node(shapes.NEGATIVE) + graph["3"] = _encode_from(["50", 0]) + return graph + + +class SaveToLibraryTests(RecordedRuns, unittest.TestCase): + """#90: the repair never relinks a file to a switched-off node's text. + + The plan decides each file through ``decide_file_prompt``; every answer + but a positive from the recorded saver or the inspector is a skip, so a + ``not_saved`` or ``not_saved_wired`` file is never written. Each file is + reported by a run without contexts (as a cached or unrecorded run is), + so only the ladder can stop it. Saver "23" is branch 2. + """ + + def setUp(self): + self.tmp = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, self.tmp, True) + self.addCleanup(assert_closed, self, self.tmp) + self.addCleanup(PromptModel.reset_schema_cache) + self.output = os.path.join(self.tmp, "output") + os.makedirs(os.path.join(self.output, "dataset")) + self.db_path = os.path.join(self.tmp, "prompts.db") + + def plan_for(self, graph): + path = os.path.join(self.output, "dataset", NAME.format(1)) + png(path, graph) + db = PromptDatabase(self.db_path) + try: + wrong = db.save_prompt( + text=TEXTS[0], prompt_hash=generate_prompt_hash(TEXTS[0]) + ) + db.link_image_to_prompt(wrong, path, metadata={}) + self.record_run( + db, self.output, [("dataset", NAME.format(1))], [], nodes=["23"] + ) + plan = tool.build_plan(db, self.output, self.db_path) + finally: + db.close_all() + (item,) = plan["rows"] + return item + + def test_an_off_node_behind_a_pass_through_is_skipped(self): + item = self.plan_for(off_behind_pass_through()) + self.assertEqual(item["action"], "skip") + self.assertEqual(item["decision"], "not_saved/none") + self.assertNotIn("target_text", item) + + def test_an_off_node_in_the_recorded_run_target_is_skipped(self): + cases = { + "off": (False, "not_saved/none"), + "wired": (["52", 0], "not_saved_wired/none"), + } + for name, (save, decision) in cases.items(): + with self.subTest(name): + if os.path.exists(self.db_path): + os.remove(self.db_path) + item = self.plan_for(off_target(save=save)) + self.assertEqual(item["action"], "skip") + self.assertEqual(item["decision"], decision) + + def test_an_off_node_on_another_branch_still_skips_the_file(self): + """The ladder judges the file's whole graph: any switched-off text + upstream of a positive makes the file not saved.""" + item = self.plan_for(off_target(branch=3)) + self.assertEqual(item["action"], "skip") + self.assertEqual(item["decision"], "not_saved/none") + + def test_an_off_node_wired_only_to_the_negative_still_repairs(self): + item = self.plan_for(off_negative_only()) + self.assertEqual(item["action"], "relink") + self.assertEqual(item["source"], "recorded-run") + self.assertEqual(item["target_text"], TEXTS[1]) + + def test_a_switched_on_node_in_the_target_still_repairs(self): + """Regression guard: a switch that is on is no reason to skip.""" + item = self.plan_for(off_target(save=True)) + self.assertEqual(item["action"], "relink") + self.assertEqual(item["target_text"], TEXTS[1]) + + +class ActionTests(unittest.TestCase): + """Only the recorded saver relinks; a certain step naming the linked + prompt keeps it; the legacy floor and the rest are skipped.""" + + def test_action_by_decision(self): + linked = generate_prompt_hash("linked") + cases = [ + (Decision("positive", "other", source="recorded-run"), "relink"), + (Decision("positive", "linked", source="recorded-run"), "keep"), + (Decision("positive", "linked", source="inspector"), "keep"), + (Decision("positive", "other", source="inspector"), "skip"), + (Decision("positive", "linked", source="legacy-parser"), "skip"), + (Decision("pair", "linked", source="resolver"), "skip"), + (Decision("never"), "skip"), + (Decision("not_saved"), "skip"), + (Decision("not_saved_wired"), "skip"), + (Decision("positive", "other", source="opted-out-negative"), "skip"), + ] + for decision, action in cases: + with self.subTest(decision.source, text=decision.text): + self.assertEqual(tool.action_of(decision, linked), action) + + +class CommandLineTests(unittest.TestCase): + def test_running_probe_reports_a_listening_port(self): + import socket + + with socket.socket() as server: + server.bind(("127.0.0.1", 0)) + server.listen(1) + port = server.getsockname()[1] + found = tool.comfyui_running("/nonexistent/comfy", port) + self.assertTrue(any(str(port) in item for item in found)) + + def test_running_probe_is_quiet_when_nothing_runs(self): + import socket + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + self.assertEqual(tool.comfyui_running("/nonexistent/comfy", port), []) + + def fake_proc(self, tmp): + proc = Path(tmp, "proc") + db = Path(tmp, "prompts.db") + db.write_bytes(b"") + Path(tmp, "prompts.db-wal").write_bytes(b"") + comfy = Path(tmp, "comfy") + comfy.mkdir() + holder = proc / "125" / "fd" + holder.mkdir(parents=True) + os.symlink(Path(tmp, "prompts.db-wal"), holder / "3") + (proc / "125" / "cmdline").write_bytes(b"python\0main.py\0") + os.symlink(comfy, proc / "125" / "cwd") + (proc / "123").mkdir() # unreadable: no fd, cmdline or cwd + (proc / "self").mkdir() # not a pid + return proc, db, comfy + + @unittest.skipIf(os.name == "nt", "symlinks need privileges on Windows") + def test_proc_scan_finds_holders_and_skips_unreadable_entries(self): + with tempfile.TemporaryDirectory() as tmp: + proc, db, comfy = self.fake_proc(tmp) + found = tool.open_by_others(str(db), proc=str(proc), own_pid=1) + self.assertEqual(len(found), 1) + self.assertIn("125", found[0]) + self.assertEqual( + tool.open_by_others(str(db), proc=str(proc), own_pid=125), [] + ) + running = tool.comfyui_running(str(comfy), 1, proc=str(proc)) + self.assertTrue(any("125" in item for item in running), running) + + def test_without_proc_the_scan_says_it_cannot_tell(self): + self.assertEqual( + tool.open_by_others("/x/prompts.db", proc="/nonexistent/proc"), + [tool.NO_PROC], + ) + + def test_missing_database_is_an_argument_error(self): + with self.assertRaises(SystemExit), contextlib.redirect_stderr(io.StringIO()): + tool.main( + ["/nonexistent/prompts.db", "--output-dir", "/tmp"], out=io.StringIO() + ) + + def test_apply_needs_the_comfyui_folder(self): + with tempfile.TemporaryDirectory() as tmp: + db = os.path.join(tmp, "prompts.db") + Path(db).write_bytes(b"") + with ( + self.assertRaises(SystemExit), + contextlib.redirect_stderr(io.StringIO()), + ): + tool.main([db, "--apply", "plan.json"], out=io.StringIO()) + + +if __name__ == "__main__": + unittest.main() diff --git a/tools/comfyui_stopped.py b/tools/comfyui_stopped.py new file mode 100644 index 00000000..c21bfec1 --- /dev/null +++ b/tools/comfyui_stopped.py @@ -0,0 +1,81 @@ +"""Is ComfyUI stopped? The checks a library repair runs before writing (#247). + +``comfyui_running`` looks for ``main.py`` processes of one ComfyUI folder +and a listener on its port. ``open_by_others`` lists the processes that hold +the library file, its ``-wal`` or its ``-shm`` open. Both read ``/proc`` +(Linux); without it ``open_by_others`` answers ``[NO_PROC]``: it cannot +tell, and the caller refuses unless the user says ComfyUI is stopped. +Entries that cannot be read (another user's process, one that just exited) +are skipped. +""" + +import os +import socket + +NO_PROC = ( + "cannot list the processes that hold the library open (no /proc): stop " + "ComfyUI and pass --i-stopped-comfyui" +) + + +def _same(path): + return os.path.normcase(os.path.realpath(path)) + + +def _pids(proc, own_pid=None): + try: + names = os.listdir(proc) + except OSError: + return [] + return [name for name in names if name.isdigit() and name != str(own_pid)] + + +def comfyui_running(comfyui_dir, port, proc="/proc"): + """``main.py`` processes running from ``comfyui_dir`` and a listener on + ``port`` (127.0.0.1), each as one line; empty when none is found.""" + found = [] + folder = _same(comfyui_dir) + for pid in _pids(proc): + try: + with open(os.path.join(proc, pid, "cmdline"), "rb") as stream: + parts = stream.read().split(b"\0") + cwd = _same(os.readlink(os.path.join(proc, pid, "cwd"))) + except OSError: + continue + if any(part.endswith(b"main.py") for part in parts) and ( + cwd == folder or any(folder.encode() in part for part in parts) + ): + found.append(f"process {pid} (main.py in {cwd})") + with socket.socket() as probe: + probe.settimeout(1) + if probe.connect_ex(("127.0.0.1", port)) == 0: + found.append(f"something listens on port {port}") + return found + + +def open_by_others(database, proc="/proc", own_pid=None): + """Processes other than ``own_pid`` (default: this one) holding + ``database``, its ``-wal`` or ``-shm`` open; ``[NO_PROC]`` without /proc.""" + if not os.path.isdir(proc): + return [NO_PROC] + own_pid = os.getpid() if own_pid is None else own_pid + targets = {_same(database + suffix): suffix for suffix in ("", "-wal", "-shm")} + found = [] + for pid in _pids(proc, own_pid): + fd_dir = os.path.join(proc, pid, "fd") + try: + fds = os.listdir(fd_dir) + except OSError: + continue + held = set() + for fd in fds: + try: + target = os.readlink(os.path.join(fd_dir, fd)) + except OSError: + continue + target = target.replace(" (deleted)", "") + if os.path.isabs(target) and _same(target) in targets: + held.add(os.path.basename(database) + targets[_same(target)]) + if held: + found.append(f"process {pid} has {', '.join(sorted(held))} open") + return found diff --git a/tools/repair_foreign_links.py b/tools/repair_foreign_links.py new file mode 100644 index 00000000..699faa96 --- /dev/null +++ b/tools/repair_foreign_links.py @@ -0,0 +1,776 @@ +"""Repair the links the old scan gave foreign dataset files (#247), on review. + +Before #247 the output scan linked each file of a foreign run (a workflow +with no PromptManager node) through the legacy parser, so every file of a +shared-prefix dataset run got the first branch's text. Recorded history +knows which save node wrote each file; the shared ladder +(``utils.file_prompt.decide_file_prompt``, scan policy) uses it +(``recorded-run``) to name the right branch. This tool judges every +``generated_images`` row of a file whose recording runs all have no +PromptManager context: + +``relink`` the ladder names a positive by the recorded saver, and its text + is not the linked prompt's +``keep`` the recorded saver or the inspector names the linked prompt +``skip`` anything else (file gone, no single recorded saver, ambiguous, + the legacy floor or another step decided) + +Dry run (the default) judges a copy of the database (SQLite backup from a +read-only connection) and writes a JSON plan and a Markdown list beside it +(``<plan stem>.list.md``). It also rehearses the apply on a second copy, to +list the prompts that would lose every link with their remaining +references. Deleting those is OFF unless the dry run is given +``--delete-orphans``; the choice is recorded in the plan and rehearsed on a +third copy. The library is never written. + +``--apply <plan.json>`` (ComfyUI stopped), only on the library the plan was +made from (its real path, recorded in the plan): + +1. refuses unless ``--comfyui-dir`` holds ``main.py``; refuses while a + ``main.py`` runs from it, anything listens on ``--port``, or any other + process holds the library, its ``-wal`` or ``-shm`` open + (``tools/comfyui_stopped.py``; without /proc only with + ``--i-stopped-comfyui``). ``--rehearsal`` skips the process and port + check, never the open-file check, and never for that folder's library; +2. saves ``<db>.pre-247-repair_<stamp>`` (or into ``--backup-dir``), never + over an existing file, and refuses unless the backup passes + ``integrity_check`` with the library's counts. Opening the library must + not change its schema (a newer checkout's migration): run the tool from a + checkout at the live code version. The running check runs once more just + before writing; +3. in ONE ``BEGIN IMMEDIATE`` transaction: re-judges every planned row and + refuses on any drift (row moved, prompt text changed, file changed, target + stored since), creates the target prompts that do not exist (category + ``scanned``, tag ``auto-scanned``, note ``Auto-scanned from <first file>``), + points each row at its target and drops its pair links (they named the old + positive). A file that already has a row for the target keeps that row and + loses the wrong one (UNIQUE(prompt_id, file_path)). Orphans are deleted only + when the plan chose it and each is still unreferenced. + +Any failed check rolls everything back and exits 2. Before/after table +counts, ``integrity_check`` and ``foreign_key_check`` are printed. + + python3 tools/repair_foreign_links.py <db> --output-dir <ComfyUI output> \\ + --plan repair_plan.json + python3 tools/repair_foreign_links.py <db> --apply repair_plan.json \\ + --comfyui-dir <ComfyUI folder> + +Run from a checkout at the live code version (after the merge: main) with +its venv. +""" + +import argparse +import datetime +import hashlib +import json +import logging +import os +import sqlite3 +import sys +import tempfile +from pathlib import Path +from types import SimpleNamespace + +_ROOT = Path(__file__).resolve().parents[1] +if str(_ROOT) not in sys.path: + sys.path.insert(0, str(_ROOT)) + +from database.maintenance import ORPHANED_PROMPTS_SQL, _delete_ids # noqa: E402 +from database.operations import PromptDatabase # noqa: E402 +from database.pair_ingestion import ( # noqa: E402 + SCAN_MARKS, + file_link_rows, + mark_created, +) +from database.run_attribution import recorded_output # noqa: E402 +from py.api import PromptManagerAPI # noqa: E402 +from tools.comfyui_stopped import ( # noqa: E402 + NO_PROC, + comfyui_running, + open_by_others, +) +from tools.review_image_links import backup_database # noqa: E402 +from utils.execution_outputs import snapshot_output_roots # noqa: E402 +from utils.file_prompt import POSITIVE, Policy, decide_file_prompt # noqa: E402 +from utils.hashing import generate_prompt_hash # noqa: E402 + +PLAN_VERSION = 1 +COMFY_PORT = 8188 +LIVE_LIBRARY = ("user", "__promptmanager", "prompts.db") +EXCERPT = 80 +COUNTED = ( + "prompts", + "generated_images", + "prompt_tags", + "image_prompt_pairs", + "workflow_documents", + "prompt_pairs", + "prompt_library_state", + "execution_run_contexts", + "tags", +) +# The ladder steps whose answer is certain enough to keep a link. +KEEP_SOURCES = ("recorded-run", "inspector") +# What must be unchanged since the dry run for a row to be written. +DRIFT_KEYS = ( + "action", + "file", + "current_prompt_id", + "current_text_sha256", + "file_size", + "file_mtime_ns", + "target_text", + "target_hash", + "target_prompt_id", +) +# Locations reported only by runs without any PromptManager context. +_FOREIGN_LOCATIONS = ( + "SELECT o.subfolder, o.filename FROM execution_run_outputs o " + "WHERE o.folder_type = 'output' AND o.root_key = ? AND o.path_style = ? " + "GROUP BY o.subfolder, o.filename " + "HAVING SUM(EXISTS (SELECT 1 FROM execution_run_contexts c " + "WHERE c.run_id = o.run_id)) = 0 " + "ORDER BY o.subfolder, o.filename" +) + + +class Refused(Exception): + """A check failed: nothing is written.""" + + +def sha256(text): + return hashlib.sha256(text.encode("utf-8")).hexdigest() + + +def metadata_reader(): + """The scan's metadata extraction (PNG text chunks, ffprobe for videos).""" + api = SimpleNamespace(logger=logging.getLogger("prompt_manager.repair_247")) + return lambda path: PromptManagerAPI._extract_comfyui_metadata(api, path) or {} + + +def foreign_rows(db, output_dir): + """The link rows of every file only foreign runs reported, by id.""" + style = "windows" if os.name == "nt" else "posix" + root_key = snapshot_output_roots(style, output_dir, output_dir).output_key + conn = db.model.get_connection() + rows = {} + for subfolder, filename in conn.execute(_FOREIGN_LOCATIONS, (root_key, style)): + path = os.path.join(output_dir, subfolder, filename) + rows.update((row["id"], row) for row in file_link_rows(conn, path)) + return [rows[key] for key in sorted(rows)] + + +class Judge: + """One file's decision per path, for all of its rows.""" + + def __init__(self, db, output_dir, read=None): + self.db = db + self.output_dir = output_dir + self.read = read or metadata_reader() + self.decisions = {} + + def decide(self, path, size): + if path not in self.decisions: + self.decisions[path] = decide_file_prompt( + self.read(path), + file_name=os.path.basename(path), + file_size=size, + in_output=True, + lookup=lambda: recorded_output(self.db, path, self.output_dir), + policy=Policy.SCAN, + ) + return self.decisions[path] + + def row(self, row): + """The plan entry of one ``generated_images`` row.""" + conn = self.db.model.get_connection() + path = row["image_path"] + (text,) = conn.execute( + "SELECT text FROM prompts WHERE id = ?", (row["prompt_id"],) + ).fetchone() + prompt_hash = generate_prompt_hash(text) + item = { + "row_id": row["id"], + "file": path, + "current_prompt_id": row["prompt_id"], + "current_text_sha256": sha256(text), + } + try: + stat = os.stat(path) + except OSError: + return {**item, "action": "skip", "reason": "file gone"} + item.update(file_size=stat.st_size, file_mtime_ns=stat.st_mtime_ns) + decision = self.decide(path, stat.st_size) + action = action_of(decision, prompt_hash) + if action == "skip": + return { + **item, + "action": "skip", + "reason": "not decided by the recorded saver", + "decision": f"{decision.kind}/{decision.source}", + } + target_hash = generate_prompt_hash(decision.text) + stored = conn.execute( + "SELECT id FROM prompts WHERE hash = ?", (target_hash,) + ).fetchone() + item.update( + target_text=decision.text, + target_hash=target_hash, + target_prompt_id=stored[0] if stored else "new", + action=action, + source=decision.source, + ) + return item + + +def action_of(decision, linked_hash): + """``relink``, ``keep`` or ``skip`` for a row linked to ``linked_hash``. + + Only the recorded saver (``recorded-run``) names a new target. A certain + step (the recorded saver or the inspector) naming the linked prompt keeps + it; the legacy floor and every other answer are skipped. + """ + if decision.kind != POSITIVE or decision.source not in KEEP_SOURCES: + return "skip" + if generate_prompt_hash(decision.text) == linked_hash: + return "keep" + return "relink" if decision.source == "recorded-run" else "skip" + + +def library_identity(path): + """How a plan names its library: the case-normalized real path.""" + return os.path.normcase(os.path.realpath(path)) + + +def build_plan(db, output_dir, source_db, read=None): + """The plan for ``db`` (a copy of ``source_db``).""" + judge = Judge(db, output_dir, read) + conn = db.model.get_connection() + (max_row_id,) = conn.execute("SELECT MAX(id) FROM generated_images").fetchone() + return { + "version": PLAN_VERSION, + "created": datetime.datetime.now().isoformat(timespec="seconds"), + "source_db": library_identity(source_db), + "max_row_id": max_row_id, + "output_dir": output_dir, + "rows": [judge.row(row) for row in foreign_rows(db, output_dir)], + "orphans": [], + "delete_orphans": False, + } + + +def blockers(conn, prompt_id, unused): + """Why ``prompt_id`` must not be deleted (empty: nothing references it).""" + found = [] + values = {"id": prompt_id, "tag": SCAN_MARKS.tags[0]} + checks = { + "linked images": "SELECT COUNT(*) FROM generated_images WHERE prompt_id=:id", + "saved pairs": "SELECT COUNT(*) FROM prompt_pairs " + "WHERE positive_prompt_id=:id OR negative_prompt_id=:id", + "recorded runs": "SELECT COUNT(*) FROM execution_run_contexts " + "WHERE positive_prompt_id=:id OR negative_prompt_id=:id", + "library state": "SELECT COUNT(*) FROM prompt_library_state " + "WHERE prompt_id=:id", + "user tags": "SELECT COUNT(*) FROM prompt_tags pt JOIN tags t " + "ON t.id=pt.tag_id WHERE pt.prompt_id=:id AND t.name != :tag", + } + for name, sql in checks.items(): + count = conn.execute(sql, values).fetchone()[0] + if count: + found.append(f"{name}: {count}") + run_count, last_used, rating, category, notes = conn.execute( + "SELECT run_count, last_used_at, rating, category, notes FROM prompts " + "WHERE id=?", + (prompt_id,), + ).fetchone() + if run_count or last_used or rating: + found.append("usage or rating") + if category not in (None, SCAN_MARKS.category): + found.append(f"category {category}") + if notes and not notes.startswith("Auto-scanned from "): + found.append("notes") + if prompt_id not in unused: + found.append("not unused by the app's rule") + return found + + +def _rejudge(conn, judge, plan): + for item in plan["rows"]: + if item["action"] == "skip": + continue + row = conn.execute( + "SELECT * FROM generated_images WHERE id = ?", (item["row_id"],) + ).fetchone() + if row is None: + raise Refused(f"row {item['row_id']}: gone") + if generate_prompt_hash(item.get("target_text") or "") != item.get( + "target_hash" + ): + raise Refused(f"row {item['row_id']}: target_text does not match its hash") + fresh = judge.row(row) + for key in DRIFT_KEYS: + if fresh.get(key) != item.get(key): + raise Refused( + f"row {item['row_id']}: {key} changed " + f"({item.get(key)!r} -> {fresh.get(key)!r})" + ) + + +def _targets(conn, db, relinks): + """``{target hash: prompt id}``, creating each "new" target once.""" + targets = {} + for item in relinks: + digest = item["target_hash"] + if digest in targets: + continue + if item["target_prompt_id"] != "new": + targets[digest] = item["target_prompt_id"] + continue + prompt_id, created = db._pair_component_outcome(conn, item["target_text"]) + if not created: + raise Refused(f"row {item['row_id']}: target prompt already stored") + mark_created(conn, db, prompt_id, SCAN_MARKS, os.path.basename(item["file"])) + targets[digest] = prompt_id + return targets + + +def _relink(conn, item, target): + """Point the row at ``target``; True when the wrong row was deleted instead.""" + row_id = item["row_id"] + correct = [ + row + for row in file_link_rows(conn, item["file"]) + if row["prompt_id"] == target and row["id"] != row_id + ] + if correct: # UNIQUE(prompt_id, file_path): the correct row stays alone + if _delete_ids(conn, "generated_images", [row_id]) != 1: + raise Refused(f"row {row_id}: delete touched no row") + return True + # A pair link records the old positive: it goes with the old link + conn.execute("DELETE FROM image_prompt_pairs WHERE image_id = ?", (row_id,)) + cursor = conn.execute( + "UPDATE generated_images SET prompt_id = ? WHERE id = ? AND prompt_id = ?", + (target, row_id, item["current_prompt_id"]), + ) + if cursor.rowcount != 1: + raise Refused(f"row {row_id}: relink touched {cursor.rowcount} rows") + return False + + +def _audit(conn, relinks): + """The prompts the relinks leave without any link, with their blockers.""" + sources = {} + for item in relinks: + sources.setdefault(item["current_prompt_id"], []).append(item) + unused = {row[0] for row in conn.execute(ORPHANED_PROMPTS_SQL)} + audit = [] + for prompt_id in sorted(sources): + if conn.execute( + "SELECT 1 FROM generated_images WHERE prompt_id = ?", (prompt_id,) + ).fetchone(): + continue + (text,) = conn.execute( + "SELECT text FROM prompts WHERE id = ?", (prompt_id,) + ).fetchone() + found = blockers(conn, prompt_id, unused) + audit.append( + { + "id": prompt_id, + "text_excerpt": " ".join(text.split())[:EXCERPT], + "text_sha256": sha256(text), + "images_before": len(sources[prompt_id]), + "blockers": found, + "safe": not found, + } + ) + return audit + + +def _delete_orphans(conn, plan): + chosen = [orphan for orphan in plan["orphans"] if orphan["safe"]] + unused = {row[0] for row in conn.execute(ORPHANED_PROMPTS_SQL)} + for orphan in chosen: + row = conn.execute( + "SELECT text FROM prompts WHERE id = ?", (orphan["id"],) + ).fetchone() + if row is None or sha256(row[0]) != orphan["text_sha256"]: + raise Refused(f"prompt {orphan['id']}: gone or text changed") + found = blockers(conn, orphan["id"], unused) + if found: + raise Refused(f"prompt {orphan['id']}: {', '.join(found)}") + ids = [orphan["id"] for orphan in chosen] + if _delete_ids(conn, "prompts", ids) != len(ids): + raise Refused("an orphan prompt was not deleted") + return len(ids) + + +def apply_plan(db, plan, read=None, before_write=None): + """Write ``plan`` in one transaction; returns ``(result, orphan audit)``. + + ``before_write()`` runs just before ``BEGIN IMMEDIATE`` (the running + check). Raises ``Refused`` (and rolls back) on any drift or failed check. + """ + judge = Judge(db, plan["output_dir"], read) + relinks = [item for item in plan["rows"] if item["action"] == "relink"] + if before_write is not None: + before_write() + with db._pair_transaction() as conn: + _rejudge(conn, judge, plan) + targets = _targets(conn, db, relinks) + dropped = sum( + _relink(conn, item, targets[item["target_hash"]]) for item in relinks + ) + audit = _audit(conn, relinks) + deleted = _delete_orphans(conn, plan) if plan.get("delete_orphans") else 0 + created = sum(1 for item in relinks if item["target_prompt_id"] == "new") + result = { + "relinked": len(relinks) - dropped, + "duplicates_dropped": dropped, + "prompts_created": len( + {i["target_hash"] for i in relinks if i["target_prompt_id"] == "new"} + ), + "rows_to_new_prompts": created, + "kept": sum(1 for item in plan["rows"] if item["action"] == "keep"), + "orphans_deleted": deleted, + } + return result, audit + + +def counts(path, readonly=True): + """Table counts plus the integrity and foreign-key checks. + + Read-only for a library; the tool's own backup is opened read-write so + a WAL file leaves no side files behind once it closes. + """ + uri = Path(os.path.abspath(path)).as_uri() + ("?mode=ro" if readonly else "") + conn = sqlite3.connect(uri, uri=True) + try: + found = { + table: conn.execute(f'SELECT COUNT(*) FROM "{table}"').fetchone()[0] + for table in COUNTED # nosec B608 - fixed table names + } + found["integrity_check"] = conn.execute("PRAGMA integrity_check").fetchone()[0] + found["foreign_key_check"] = len( + conn.execute("PRAGMA foreign_key_check").fetchall() + ) + return found + finally: + conn.close() + + +def _print_counts(before, after, out): + out.write(f"{'':24}{'before':>12}{'after':>12}\n") + for key in before: + out.write(f"{key:24}{before[key]!s:>12}{after[key]!s:>12}\n") + + +def _summary(plan): + actions = [item["action"] for item in plan["rows"]] + return " ".join( + f"{name}: {actions.count(name)}" for name in ("relink", "keep", "skip") + ) + + +def _cell(text, limit=EXCERPT): + """One table cell: whitespace folded, cut to ``limit`` (None: whole).""" + folded = " ".join(str(text).split()) + return (folded if limit is None else folded[:limit]).replace("|", "\\|") + + +def _relative(path, root): + try: + return os.path.relpath(path, root) + except ValueError: # another drive on Windows + return path + + +def markdown(plan, prompt_texts): + """The maintainer's list of the plan.""" + root = plan["output_dir"] + rel = lambda path: _relative(path, root) # noqa: E731 + rows = plan["rows"] + lines = [ + "# #247 repair plan (dry run)", + "", + f"Created {plan['created']}; output folder `{root}`. {_summary(plan)}.", + "", + f"## Relink ({sum(i['action'] == 'relink' for i in rows)})", + "", + "| row | file | linked prompt | linked text | target | target hash " + "| target text |", + "|---:|---|---:|---|---|---|---|", + ] + for item in (i for i in rows if i["action"] == "relink"): + lines.append( + f"| {item['row_id']} | {_cell(rel(item['file']), None)} | " + f"{item['current_prompt_id']} | " + f"{_cell(prompt_texts.get(item['current_prompt_id'], ''))} | " + f"{item['target_prompt_id']} | `{item['target_hash'][:12]}` | " + f"{_cell(item['target_text'])} |" + ) + lines += ["", f"## Keep ({sum(i['action'] == 'keep' for i in rows)})", ""] + lines += ["| row | file | prompt |", "|---:|---|---:|"] + for item in (i for i in rows if i["action"] == "keep"): + lines.append( + f"| {item['row_id']} | {_cell(rel(item['file']), None)} | " + f"{item['current_prompt_id']} |" + ) + lines += ["", f"## Skip ({sum(i['action'] == 'skip' for i in rows)})", ""] + lines += ["| row | file | prompt | reason |", "|---:|---|---:|---|"] + for item in (i for i in rows if i["action"] == "skip"): + reason = item["reason"] + ( + f" ({item['decision']})" if item.get("decision") else "" + ) + lines.append( + f"| {item['row_id']} | {_cell(rel(item['file']), None)} | " + f"{item['current_prompt_id']} | {reason} |" + ) + choice = "ON (--delete-orphans)" if plan["delete_orphans"] else "OFF" + lines += [ + "", + f"## Prompts that would lose every link ({len(plan['orphans'])})", + "", + f"Deleting the safe ones: {choice}.", + "", + "| id | text | images before | blockers | safe to delete |", + "|---:|---|---:|---|---|", + ] + for orphan in plan["orphans"]: + lines.append( + f"| {orphan['id']} | {_cell(orphan['text_excerpt'])} | " + f"{orphan['images_before']} | {', '.join(orphan['blockers']) or 'none'} | " + f"{'yes' if orphan['safe'] else 'no'} |" + ) + simulated = plan.get("simulated") + if simulated: + lines += ["", "## Rehearsed on the dry run's copy", ""] + lines += ["| table | before | after |", "|---|---:|---:|"] + for key, value in simulated["before"].items(): + lines.append(f"| {key} | {value} | {simulated['after'][key]} |") + return "\n".join(lines) + "\n" + + +def _texts(db): + return dict( + db.model.get_connection().execute( + "SELECT id, text FROM prompts WHERE id IN (SELECT prompt_id " + "FROM generated_images)" + ) + ) + + +def _rehearse(source, scratch, name, plan): + """``apply_plan`` on a fresh backup of ``source``: (counts, result, audit).""" + copy = os.path.join(scratch, name) + backup_database(source, copy) + db = PromptDatabase(copy) + try: + before = counts(copy) + result, audit = apply_plan(db, plan) + finally: + db.close_all() + return {"before": before, "after": counts(copy), "result": result}, audit + + +def dry_run(args, out): + with tempfile.TemporaryDirectory() as scratch: + copy = os.path.join(scratch, "judge.db") + backup_database(args.database, copy) + db = PromptDatabase(copy) + try: + plan = build_plan(db, os.path.abspath(args.output_dir), args.database) + texts = _texts(db) + finally: + db.close_all() + simulated, audit = _rehearse(args.database, scratch, "relinks.db", plan) + plan.update(orphans=audit, delete_orphans=bool(args.delete_orphans)) + if plan["delete_orphans"]: + # The plan as the apply will write it, deletes included + simulated, _audit = _rehearse(args.database, scratch, "deletes.db", plan) + plan["simulated"] = simulated + plan_path = Path(args.plan) + plan_path.write_text( + json.dumps(plan, indent=1, ensure_ascii=False) + "\n", encoding="utf-8" + ) + list_path = plan_path.with_name(plan_path.stem + ".list.md") + list_path.write_text(markdown(plan, texts), encoding="utf-8") + out.write(f"{_summary(plan)}\n") + out.write( + f"Prompts that would lose every link: {len(audit)} " + f"({sum(o['safe'] for o in audit)} safe to delete; deleting " + f"{'ON' if plan['delete_orphans'] else 'OFF'})\n" + ) + out.write(f"Plan: {plan_path}\nList: {list_path}\n") + out.write("Dry run: nothing was changed.\n") + return 0 + + +def _is_live(database, comfyui_dir): + live = os.path.join(comfyui_dir, *LIVE_LIBRARY) + return library_identity(database) == library_identity(live) + + +def _stamp(): + return datetime.datetime.now().strftime("%Y%m%d_%H%M%S") + + +def schema_of(path): + """``sqlite_master`` and ``user_version`` from a read-only connection.""" + uri = Path(os.path.abspath(path)).as_uri() + "?mode=ro" + conn = sqlite3.connect(uri, uri=True) + try: + master = conn.execute( + "SELECT type, name, tbl_name, sql FROM sqlite_master ORDER BY type, name" + ).fetchall() + return master, conn.execute("PRAGMA user_version").fetchone()[0] + finally: + conn.close() + + +def running_reasons(args, probe, handles): + """Why writing must not start now (empty: go ahead). + + Another process holding the library open always counts, also in a + rehearsal; ComfyUI's processes and port count unless rehearsing. + """ + held = [ + reason + for reason in handles(args.database) + if not (reason == NO_PROC and args.i_stopped_comfyui) + ] + return held + ([] if args.rehearsal else probe(args.comfyui_dir, args.port)) + + +def _preflight(args, plan, probe, handles): + """The refusals before any backup or write, as a message (None: go on).""" + if plan.get("version") != PLAN_VERSION: + return "not a plan of this tool" + if args.rehearsal and _is_live(args.database, args.comfyui_dir): + return f"--rehearsal is for a copy; this is the library of {args.comfyui_dir}" + if plan.get("source_db") != library_identity(args.database): + return f"this plan was made for {plan.get('source_db')}" + if not os.path.isfile(os.path.join(args.comfyui_dir, "main.py")): + return f"no main.py in {args.comfyui_dir}: not a ComfyUI folder" + reasons = running_reasons(args, probe, handles) + if reasons: + return "ComfyUI may be running: " + "; ".join(reasons) + return None + + +def _backup(args, before): + """The verified backup's path; raises ``Refused``.""" + folder = args.backup_dir or os.path.dirname(os.path.abspath(args.database)) + name = f"{os.path.basename(args.database)}.pre-247-repair_{_stamp()}" + backup = os.path.join(folder, name) + if os.path.exists(backup): + raise Refused(f"the backup {backup} already exists") + backup_database(args.database, backup) + saved = counts(backup, readonly=False) + if saved != before or saved["integrity_check"] != "ok": + raise Refused(f"the backup {backup} does not match the library") + return backup + + +def _open_unchanged(path): + """``PromptDatabase(path)``; raises ``Refused`` when opening it changed + its schema (a migration of a newer checkout, committed on open).""" + schema = schema_of(path) + db = PromptDatabase(path) + db.model.get_connection() + if schema_of(path) != schema: + db.close_all() + raise Refused( + "opening the library with this checkout changed its schema (that " + "change is committed); run the tool from a checkout at the live " + "code version" + ) + return db + + +def apply(args, out, probe, handles): + with open(args.apply, encoding="utf-8") as stream: + plan = json.load(stream) + refusal = _preflight(args, plan, probe, handles) + if refusal: + out.write(f"REFUSED: {refusal}\n") + return 2 + if args.rehearsal: + out.write("(rehearsal on a copy: ComfyUI's process and port not checked)\n") + before = counts(args.database) + try: + backup = _backup(args, before) + except Refused as error: + out.write(f"REFUSED: {error}\n") + return 2 + out.write(f"Backup: {backup}\n") + + def still_stopped(): + reasons = running_reasons(args, probe, handles) + if reasons: + raise Refused("ComfyUI may be running: " + "; ".join(reasons)) + + try: + db = _open_unchanged(args.database) + except Refused as error: + out.write(f"REFUSED: {error}. Backup: {backup}\n") + return 2 + try: + result, _audit = apply_plan(db, plan, before_write=still_stopped) + except Exception as error: + out.write( + f"REFUSED (rolled back, nothing changed): {error}. Backup: {backup}\n" + ) + return 2 + finally: + db.close_all() + after = counts(args.database) + _print_counts(before, after, out) + out.write(f"Applied: {json.dumps(result, sort_keys=True)}\n") + if after["integrity_check"] != "ok" or after["foreign_key_check"]: + out.write(f"WARNING: integrity or foreign-key check failed; restore {backup}\n") + return 1 + return 0 + + +def main(argv=None, out=sys.stdout, probe=comfyui_running, handles=open_by_others): + parser = argparse.ArgumentParser(description=__doc__.split("\n", 1)[0]) + parser.add_argument("database", help="the library database (prompts.db)") + parser.add_argument("--output-dir", help="ComfyUI's output folder (dry run)") + parser.add_argument( + "--plan", default="repair_foreign_links_plan.json", help="plan to write" + ) + parser.add_argument( + "--delete-orphans", + action="store_true", + help="record in the plan that the safe orphan prompts are deleted", + ) + parser.add_argument("--apply", metavar="PLAN", help="write this reviewed plan") + parser.add_argument("--comfyui-dir", help="the ComfyUI folder (apply)") + parser.add_argument("--backup-dir", help="where the backup goes (apply)") + parser.add_argument("--port", type=int, default=COMFY_PORT) + parser.add_argument( + "--rehearsal", + action="store_true", + help="apply to a copy while ComfyUI runs (never its own library)", + ) + parser.add_argument( + "--i-stopped-comfyui", + action="store_true", + help="without /proc: say ComfyUI is stopped (the port is still checked)", + ) + args = parser.parse_args(argv) + if not os.path.isfile(args.database): + parser.error(f"no database at {args.database}") + if args.apply: + if not args.comfyui_dir: + parser.error("--apply needs --comfyui-dir (the running check)") + return apply(args, out, probe, handles) + if not args.output_dir: + parser.error("the dry run needs --output-dir") + if Path(args.plan).suffix.lower() == ".md": + parser.error("--plan is the JSON plan; the list is written beside it") + return dry_run(args, out) + + +if __name__ == "__main__": + sys.exit(main()) From b6e04394218c9a7bfd5d4aba7e5657731047507a Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Thu, 29 Oct 2026 12:58:34 +0000 Subject: [PATCH 4/7] ci(browser): run the Firefox and WebKit page workflows in three shards (#473) (#474) Shard 1/2 of each browser had reached its 18-minute step limit: --shard splits by test count, so the slow concept specs all land in shard 1. Three shards per browser, with the limits unchanged. Required contexts do not change: only the viewerjs aggregate is required, and it reads the matrix job's result over every shard. Closes #473 --- .github/workflows/viewerjs-check.yml | 13 +++++++++---- tests/vendor/ci-workflow.test.mjs | 10 +++++++--- 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/.github/workflows/viewerjs-check.yml b/.github/workflows/viewerjs-check.yml index 4a87231d..d5c83424 100644 --- a/.github/workflows/viewerjs-check.yml +++ b/.github/workflows/viewerjs-check.yml @@ -36,6 +36,11 @@ concurrency: # 12m53s, node units and the viewer suite before 11.8m of workflows), so coverage is the tight one # in its 18 minutes. This holds only when the npm cache hits: a cold cache adds up to 5m to npm ci, # and then the job limit can still end the job before the step limit does. +# #473: firefox/webkit shard 1/2 reached its 18-minute step limit (queue run 40866252975: webkit 622 tests +# passed in 17m56s, 48.5 worker-minutes vs 28.0 in shard 2/2; recent shard 1/2 jobs 13.7m-18.9m, shard 2/2 +# 7.3m-12.2m), because --shard splits by test count and the slow concept specs sort into shard 1. The page +# workflows now run in three shards per browser with the limits unchanged; the per-shard times are +# re-measured after three green main runs. jobs: frontend-coverage: name: frontend coverage (shard ${{ matrix.shard }}/3) @@ -175,7 +180,7 @@ jobs: retention-days: 14 browser-workflows: - name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/2) + name: page workflows (${{ matrix.browser }}, shard ${{ matrix.shard }}/3) # Merge queue, main and manual runs only (#420); pull requests run the Chromium/touch coverage shards. if: github.event_name != 'pull_request' runs-on: ubuntu-latest @@ -194,7 +199,7 @@ jobs: fail-fast: false matrix: browser: [firefox, webkit] - shard: [1, 2] + shard: [1, 2, 3] steps: - uses: actions/checkout@v7 # The image's own Python 3.12 and Node 24: setup-python/setup-node install the runner's toolcache @@ -221,9 +226,9 @@ jobs: path: ~/.npm key: npm-${{ runner.os }}-playwright-v1.64.1-${{ hashFiles('tools/vendor/package-lock.json') }} - run: npm ci --prefix tools/vendor --ignore-scripts - - name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/2 + - name: Page workflows in ${{ matrix.browser }}, shard ${{ matrix.shard }}/3 timeout-minutes: 18 - run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/2 + run: npm run --prefix tools/vendor test:workflows -- --project ${{ matrix.browser }} --shard ${{ matrix.shard }}/3 # Traces and the blob report (test-results/blob/) of a failed, timed-out or cancelled step. - name: Upload browser traces if: always() diff --git a/tests/vendor/ci-workflow.test.mjs b/tests/vendor/ci-workflow.test.mjs index 0e3d0c62..c146bb58 100644 --- a/tests/vendor/ci-workflow.test.mjs +++ b/tests/vendor/ci-workflow.test.mjs @@ -212,11 +212,15 @@ test('every browser job sets PM_TEST_WORKERS and runs in the pinned image', () = assert.match(mergeJob(), PINNED_IMAGE, 'the merge sees the same workspace paths as the shards'); }); -test('page workflows run each browser in two shards', () => { +// #473: two shards split by test count put the slow concept specs in shard 1, which reached the 18-minute step limit. +test('page workflows run each browser in three shards', () => { const workflows = allJobs.get('browser-workflows'); assert.deepEqual(matrixList(workflows, 'browser'), ['firefox', 'webkit']); - assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2']); - assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/2/); + assert.deepEqual(matrixList(workflows, 'shard'), ['1', '2', '3']); + assert.match(workflows, /test:workflows -- --project \$\{\{ matrix\.browser \}\} --shard \$\{\{ matrix\.shard \}\}\/3/); + assert.match(workflows, /^ {4}name: page workflows \(\$\{\{ matrix\.browser \}\}, shard \$\{\{ matrix\.shard \}\}\/3\)$/m); + assert.match(workflows, /name: Page workflows in \$\{\{ matrix\.browser \}\}, shard \$\{\{ matrix\.shard \}\}\/3$/m); + assert.doesNotMatch(workflows, /\/2\b/, 'no step still names two shards'); assert.match(workflows, /^ {6}fail-fast: false$/m); assert.match(workflows, /name: browser-traces-\$\{\{ matrix\.browser \}\}-shard-\$\{\{ matrix\.shard \}\}/); }); From 89d94cdb2b29ef7130a4199b64626dbf61bd6962 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Thu, 29 Oct 2026 13:25:04 +0000 Subject: [PATCH 5/7] fix(history): join case-variant raw text to the history of the prompt it counts toward (#460) (#471) * fix(history): join outer-whitespace raw text to the history of the prompt it saves (#438) The Library stores text.strip() while a run keeps its exact raw text. The history window, run detail scope id, newest pair run and prompt hints now compare the raw text's stored form (one rule, utils/hashing.py, mirrored in SQLite with the same code points). Hints also keep matching the exact raw text, so older rows that kept outer whitespace stay hinted and protected. * fix(history): join case-variant raw text to the history of the prompt it counts toward (#460) The prompt hash folds case, so a run of "A Lighthouse" counts toward the saved "a lighthouse" but never joined its history. History now compares the hash's own key (prompt_key = strip + str.lower()); SQL calls Python for it through pm_prompt_key/pm_prompt_hash registered on every managed connection, since SQLite's lower() folds ASCII only. Hints match by text first (#438) and fall back to the dedup hash only when no text matches, so no existing hint becomes ambiguous. A history position from before this change still resumes. * test(history): a real-host case-variant re-queue joins the prompt's history (#460) --- database/execution_contexts.py | 74 +++++++++++++-------- database/health.py | 10 +-- database/history_reads.py | 39 ++++++----- database/maintenance.py | 6 ++ database/models.py | 8 +++ tests/comfyui/execution-history.spec.mjs | 10 +++ tests/test_api_history.py | 8 ++- tests/test_execution_context_database.py | 82 ++++++++++++++++++++++++ tests/test_execution_history_hooks.py | 25 ++++++++ tests/test_hashing.py | 53 +++++++++++++++ tests/test_history_reads.py | 78 +++++++++++++++++++++- utils/hashing.py | 34 ++++++++-- 12 files changed, 368 insertions(+), 59 deletions(-) diff --git a/database/execution_contexts.py b/database/execution_contexts.py index aeb4b801..680da4d8 100644 --- a/database/execution_contexts.py +++ b/database/execution_contexts.py @@ -7,7 +7,11 @@ from typing import Callable, ContextManager, Iterator try: from ..utils.execution_history_events import RunFinish, encode_command - from ..utils.hashing import stored_prompt_text, stored_prompt_text_sql + from ..utils.hashing import ( + generate_prompt_hash, + stored_prompt_text, + stored_prompt_text_sql, + ) from ..utils.execution_contexts import ( CAPTURE_REASONS, ContextBundle, @@ -16,7 +20,11 @@ try: ) except ImportError: from utils.execution_history_events import RunFinish, encode_command - from utils.hashing import stored_prompt_text, stored_prompt_text_sql + from utils.hashing import ( + generate_prompt_hash, + stored_prompt_text, + stored_prompt_text_sql, + ) from utils.execution_contexts import ( CAPTURE_REASONS, ContextBundle, @@ -149,9 +157,12 @@ def _role_hints(role: str) -> str: # Set form of the read-time prompt-hint rule in get_execution_contexts(): # a stored role hint is live only in the current storage epoch, and only when -# exactly one prompt has its raw snapshot text or that text's stored form -# (outer whitespace stripped, #438), and that is the hinted one (_prompt_hint). -# Evaluated once per statement, never per prompt row. +# the hinted prompt is the one match of _prompt_hint: by text (the raw +# snapshot text or its stored form, outer whitespace stripped, #438), or, only +# when no prompt has either text, by dedup hash (case folded too, #460). +# The hash is a fallback, so a hint #438 kept is never made ambiguous. +# Needs register_prompt_functions (pm_prompt_hash). The hint rows are the +# outer loop (CROSS JOIN), so each prompts lookup is by index. # Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH. LIVE_HINT_CTES = ( "stored_hints(prompt_id, raw, stored) AS (" @@ -160,7 +171,12 @@ LIVE_HINT_CTES = ( + _role_hints("negative") + "), hint_matches(prompt_id, raw, stored, match_id) AS (" "SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h " - "JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored)" + "CROSS JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored) " + "UNION ALL " + "SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h " + "CROSS JOIN prompts p ON p.hash = pm_prompt_hash(h.raw) " + "WHERE NOT EXISTS (SELECT 1 FROM prompts q " + "WHERE q.text COLLATE BINARY IN (h.raw, h.stored))" "), live_hint_prompts(prompt_id) AS (" "SELECT prompt_id FROM hint_matches GROUP BY prompt_id, raw, stored " "HAVING COUNT(*) = 1 AND MIN(match_id) = prompt_id)" @@ -169,20 +185,32 @@ LIVE_HINT_PROMPT_IDS_SQL = ( "WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts" ) +_BY_TEXT = "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2" + + +def _text_matches(conn: sqlite3.Connection, raw: str) -> list: + return [row[0] for row in conn.execute(_BY_TEXT, (raw, stored_prompt_text(raw)))] + def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None: - """The one prompt whose text is ``raw`` or the stored form of it (#438). + """The one prompt that stores ``raw`` (#438), else the one it counts toward. - The Library stores ``text.strip()``; the exact form keeps hints of older - rows whose stored text kept outer whitespace. + By text first: ``raw`` or ``raw.strip()`` (the exact form keeps hints of + older rows that kept outer whitespace). Only when no prompt has either + text, by the dedup hash, which also folds case (#460). """ if raw is None: return None - rows = conn.execute( - "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2", - (raw, stored_prompt_text(raw)), - ).fetchall() - return rows[0][0] if len(rows) == 1 else None + matches = _text_matches(conn, raw) + if not matches: + matches = [ + row[0] + for row in conn.execute( + "SELECT id FROM prompts WHERE hash=? LIMIT 2", + (generate_prompt_hash(raw),), + ) + ] + return matches[0] if len(matches) == 1 else None def _hints( @@ -191,24 +219,16 @@ def _hints( pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative) stored = None if negative is None else stored_prompt_text(negative) pair_id = None + # A known negative that names no prompt because several share its text negative_ambiguous = ( - stored is not None - and neg_id is None - and conn.execute( - "SELECT 1 FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 1", - (negative, stored), - ).fetchone() - is not None + stored is not None and neg_id is None and bool(_text_matches(conn, negative)) ) if pos_id is not None and stored is not None and not negative_ambiguous: rows = conn.execute( - "SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON " - "n.id=pp.negative_prompt_id " + "SELECT pp.id FROM prompt_pairs pp " "WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND " - "?='') " - "OR (pp.negative_prompt_id=? AND n.text COLLATE BINARY IN (?,?))) " - "LIMIT 2", - (pos_id, stored, neg_id, negative, stored), + "?='') OR pp.negative_prompt_id=?) LIMIT 2", + (pos_id, stored, neg_id), ).fetchall() if len(rows) == 1: pair_id = rows[0][0] diff --git a/database/health.py b/database/health.py index e17a2022..58f0ddd7 100644 --- a/database/health.py +++ b/database/health.py @@ -23,10 +23,10 @@ from pathlib import PurePosixPath, PureWindowsPath from urllib.parse import quote try: - from ..utils.hashing import generate_prompt_hash + from ..utils.hashing import register_prompt_functions from .models import normalize_image_path except ImportError: - from utils.hashing import generate_prompt_hash + from utils.hashing import register_prompt_functions from database.models import normalize_image_path logger = logging.getLogger("prompt_manager.database.health") @@ -125,12 +125,8 @@ def _foreign_keys(conn, _roots): ) -def _prompt_hash(text): - return generate_prompt_hash(text) if isinstance(text, str) else None - - def _canonical_hashes(conn, _roots): - conn.create_function("pm_prompt_hash", 1, _prompt_hash, deterministic=True) + register_prompt_functions(conn) # pm_prompt_hash, shared with history (#460) return _counted( f"Prompt {row[0]}: the stored hash does not match its text" for row in conn.execute( diff --git a/database/history_reads.py b/database/history_reads.py index b803f55e..08de0ca5 100644 --- a/database/history_reads.py +++ b/database/history_reads.py @@ -16,9 +16,9 @@ from .execution_outputs import _capture, _REFERENCE_COLUMNS from .library import _ELIGIBLE try: - from ..utils.hashing import stored_prompt_text, stored_prompt_text_sql + from ..utils.hashing import prompt_key, stored_prompt_text except ImportError: - from utils.hashing import stored_prompt_text, stored_prompt_text_sql + from utils.hashing import prompt_key, stored_prompt_text RunKey = tuple[int, str, int, str] @@ -115,12 +115,12 @@ _MATCHING = ( ) -def _scope_text_ids(conn: sqlite3.Connection, scope: str) -> str: - """Every recorded raw text whose stored form is ``scope``, as a JSON array. +def _scope_text_ids(conn: sqlite3.Connection, key: str) -> str: + """Every recorded raw text whose prompt key is ``key``, as a JSON array. - The Library stores ``text.strip()`` while a run keeps its exact raw text, - so a raw text joins the scope of the prompt it saves (#438). The values - are compared, never only their digests. + A run keeps its exact raw text, while the Library saves and counts it as + the prompt with its key (outer whitespace #438, case #460). The values are + compared, never only their digests; ``pm_prompt_key`` is Python's key. """ return json.dumps( [ @@ -130,16 +130,16 @@ def _scope_text_ids(conn: sqlite3.Connection, scope: str) -> str: "SELECT t.text_id FROM (SELECT DISTINCT raw_text_id AS text_id " "FROM execution_prompt_snapshots) s CROSS JOIN " "execution_context_texts t WHERE t.text_id=s.text_id AND " - f"{stored_prompt_text_sql('t.text_value')}=? COLLATE BINARY", - (scope,), + "pm_prompt_key(t.text_value)=? COLLATE BINARY", + (key,), ) ] ) def _scope_text_id(raw: str | None) -> str | None: - """The scope digest of a raw text: that of the text the Library stores.""" - return None if raw is None else content_digest("text", (stored_prompt_text(raw),)) + """The scope digest of a raw text: that of its prompt key.""" + return None if raw is None else content_digest("text", (prompt_key(raw),)) def _limit(limit: int) -> None: @@ -290,7 +290,14 @@ class HistoryReadOperations: return None text = saved["text"] raw_id = _scope_text_id(text) - if after is not None and after.raw_text_id != raw_id: + # A position from before #460 names the scope by the stripped text + # alone; it is the same scope, so a page open across the upgrade resumes. + legacy_id = ( + None + if text is None + else content_digest("text", (stored_prompt_text(text),)) + ) + if after is not None and after.raw_text_id not in (raw_id, legacy_id): raise ValueError("History position text scope changed") group = { "positive_id": positive_id, @@ -309,7 +316,7 @@ class HistoryReadOperations: ) if raw_id is None: return result - scope = _scope_text_ids(conn, stored_prompt_text(text)) + scope = _scope_text_ids(conn, prompt_key(text)) where = f"r.run_id IN ({_MATCHING})" params = [scope] if after is not None: @@ -465,7 +472,7 @@ class HistoryReadOperations: if saved["text"] is None: return {"run": None} epoch = self.get_execution_epoch() - stored = stored_prompt_text(saved["text"]) + key = prompt_key(saved["text"]) candidates = conn.execute( "SELECT r.run_id,rc.anchor_id,rc.context_id," "c.positive_snapshot_id,c.negative_snapshot_id " @@ -477,7 +484,7 @@ class HistoryReadOperations: "AND rc.reference_epoch=? " f"ORDER BY {_ORDER},rc.anchor_id COLLATE BINARY LIMIT ?", ( - _scope_text_ids(conn, stored), + _scope_text_ids(conn, key), pair_id, epoch, _PAIR_RUN_CANDIDATES, @@ -489,7 +496,7 @@ class HistoryReadOperations: if ( positive is None or positive["raw"] is None - or stored_prompt_text(positive["raw"]) != stored + or prompt_key(positive["raw"]) != key ): continue hints = _hints(conn, positive["raw"], negative["raw"] if negative else None) diff --git a/database/maintenance.py b/database/maintenance.py index 76a7abdc..a76d2d9c 100644 --- a/database/maintenance.py +++ b/database/maintenance.py @@ -44,6 +44,11 @@ from .health import BATCH_ROWS, link_states, readable_roots, readonly_uri from .pairs import PairDependencyError from .workflow_documents import documents_of_rows, release_documents +try: + from ..utils.hashing import register_prompt_functions +except ImportError: + from utils.hashing import register_prompt_functions + MAINTENANCE_OPERATIONS = ( "duplicate_prompts", "optimize", @@ -289,6 +294,7 @@ def _read_snapshot(db_path, read): conn = sqlite3.connect(readonly_uri(db_path), uri=True) try: conn.row_factory = sqlite3.Row + register_prompt_functions(conn) # LIVE_HINT_CTES calls them (#460) conn.execute("BEGIN") # one read snapshot for the whole plan return read(conn) finally: diff --git a/database/models.py b/database/models.py index 805d665f..0ef8d35e 100644 --- a/database/models.py +++ b/database/models.py @@ -38,6 +38,11 @@ from .review_schema import ( review_without_annotations_error, ) from .saved_views import migrate_saved_views, saved_views_schema_error + +try: + from ..utils.hashing import register_prompt_functions +except ImportError: + from utils.hashing import register_prompt_functions from .image_categories import image_categories_schema_error, migrate_image_categories from .workflow_documents import ( DOCUMENT_ID_COLUMNS, @@ -153,12 +158,15 @@ def configure_connection(conn: sqlite3.Connection) -> None: (common on Windows) let other writers' polling busy handlers starve past BUSY_TIMEOUT_SECONDS and fail with "database is locked" (#242). Without WAL (for example an in-memory database) SQLite's FULL default is kept. + It also registers the prompt-key SQL functions (register_prompt_functions). """ journal_mode = conn.execute("PRAGMA journal_mode = WAL").fetchone()[0] conn.execute("PRAGMA foreign_keys = ON") conn.execute(f"PRAGMA busy_timeout = {BUSY_TIMEOUT_SECONDS * 1000}") if str(journal_mode).lower() == "wal": conn.execute("PRAGMA synchronous = NORMAL") + # History reads and the live-hint rule call Python's prompt key (#460) + register_prompt_functions(conn) class PromptModel: diff --git a/tests/comfyui/execution-history.spec.mjs b/tests/comfyui/execution-history.spec.mjs index eab2b16e..ec579e6a 100644 --- a/tests/comfyui/execution-history.spec.mjs +++ b/tests/comfyui/execution-history.spec.mjs @@ -314,6 +314,16 @@ test('fresh registered PM variants preserve exact raw whitespace and known empty const runId = (await readExecutionRuns(execution.databasePath, id)).rows[0].run_id; const response = await execution.request.get(`/prompt_manager/library/${value.positive_prompt_id}/history`);expect(response.ok()).toBe(true); expect((await response.json()).items.map(item => item.run_id)).toContain(runId); + if (kind !== 'PromptManager') continue; + // #460: an upper-case re-queue counts toward the same prompt and joins its history + const upper = await submit(execution.request, recipeGraph({ kind, text: `${kind.toUpperCase()} E\u0301` })); + await history(execution.request, upper); + const variant = (await capture(execution, upper)).contexts[0]; + expect(variant.positive.raw).toBe(`${kind.toUpperCase()} E\u0301`); + expect(variant.positive_prompt_id, 'the case variant names the same prompt').toBe(value.positive_prompt_id); + const variantRun = (await readExecutionRuns(execution.databasePath, upper)).rows[0].run_id; + const scoped = await execution.request.get(`/prompt_manager/library/${value.positive_prompt_id}/history`); + expect((await scoped.json()).items.map(item => item.run_id)).toContain(variantRun); } }); diff --git a/tests/test_api_history.py b/tests/test_api_history.py index dc952332..f42e4b60 100644 --- a/tests/test_api_history.py +++ b/tests/test_api_history.py @@ -18,6 +18,8 @@ from unittest.mock import patch from aiohttp import FormData, web +from utils.hashing import generate_prompt_hash + try: from test_api_images import ImageAPITestCase except ImportError: @@ -191,7 +193,11 @@ class HistoryAPITests(ImageAPITestCase): context["evidence"]["positive"]["raw"]["provenance"], "recorded" ) with self.api.db.model.get_connection() as conn: - conn.execute("UPDATE prompts SET text='edited' WHERE id=?", (positive,)) + # An edit changes the text and its dedup hash together (#460) + conn.execute( + "UPDATE prompts SET text='edited',hash=? WHERE id=?", + (generate_prompt_hash("edited"), positive), + ) _, changed = await self.detail(start.run_id) self.assertEqual(changed["contexts"][0]["positive"], context["positive"]) self.assertFalse(changed["contexts"][0]["navigation"]["positive_available"]) diff --git a/tests/test_execution_context_database.py b/tests/test_execution_context_database.py index b9c7325f..b811906b 100644 --- a/tests/test_execution_context_database.py +++ b/tests/test_execution_context_database.py @@ -15,6 +15,7 @@ from database import execution_contexts from database.execution_history import HistoryEpochChanged from database.execution_context_schema import context_schema_error from database.execution_contexts import LIVE_HINT_PROMPT_IDS_SQL +from utils.hashing import generate_prompt_hash from database.execution_history_schema import rotate_history_epoch from utils.execution_contexts import ( ContextBundle, @@ -466,7 +467,11 @@ class ContextDatabaseTests(unittest.TestCase): with conn: conn.execute("DELETE FROM prompts WHERE id=3") self.assertEqual(assert_same(), {1, 2, 4}) + # A case-only edit keeps the prompt's hash, so the run still counts + # toward it (#460); another text drops the hint. db.update_prompt_text(4, "Other") + self.assertEqual(assert_same(), {1, 2, 4}) + db.update_prompt_text(4, "another") self.assertEqual(assert_same(), {1, 2}) with conn: conn.execute("INSERT INTO prompts(id,text) VALUES(5,'none')") @@ -522,6 +527,83 @@ class ContextDatabaseTests(unittest.TestCase): live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} self.assertEqual(live, {1, 2}) + def test_hints_follow_the_hash_for_case_variant_raw_text(self): + # #460: the run names the prompt the Library counted it toward. + db = self.database() + first = db.save_prompt( + "\u00e9t\u00e9 positive", + prompt_hash=generate_prompt_hash("\u00e9t\u00e9 positive"), + ) + second = db.save_prompt( + "negative \u03c3", prompt_hash=generate_prompt_hash("negative \u03c3") + ) + with db.model.get_connection() as conn: + conn.execute( + "INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) " + "VALUES(7,?,?)", + (first, second), + ) + start = self.capture( + db, 0, bundle("NEGATIVE \u03a3", positive="\u00c9T\u00c9 Positive") + ) + context = db.get_execution_contexts(start.run_id)["contexts"][0] + self.assertEqual( + ( + context["positive_prompt_id"], + context["negative_prompt_id"], + context["pair_id"], + ), + (first, second, 7), + ) + conn = db.model.get_connection() + live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + self.assertEqual(live, {first, second}) + # A prompt that only shares SQLite's ASCII lower() is not a match + other = db.save_prompt("other", prompt_hash=generate_prompt_hash("other")) + start = self.capture(db, 1, bundle("x", positive="\u00d6THER")) + context = db.get_execution_contexts(start.run_id)["contexts"][0] + self.assertIsNone(context["positive_prompt_id"]) + self.assertNotIn( + other, {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + ) + + def test_hash_is_only_a_fallback_so_a_text_hint_stays_unambiguous(self): + # #460 review: a hash match must never make a #438 text hint ambiguous, + # or its prompt would lose the recorded-run prune protection. + db = self.database() + conn = db.model.get_connection() + with conn: + # A legacy row without a hash, and a case variant with the hash + conn.execute("INSERT INTO prompts(id,text) VALUES(1,'a lighthouse')") + conn.execute( + "INSERT INTO prompts(id,text,hash) VALUES(2,'A Lighthouse',?)", + (generate_prompt_hash("a lighthouse"),), + ) + exact = self.capture(db, 0, bundle("x", positive="a lighthouse")) + variant = self.capture(db, 1, bundle("x", positive="A LIGHTHOUSE")) + hints = [ + db.get_execution_contexts(start.run_id)["contexts"][0]["positive_prompt_id"] + for start in (exact, variant) + ] + self.assertEqual(hints, [1, 2]) + live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + self.assertEqual(live, {1, 2}) + self.assertEqual(db.prune_orphaned_prompts(), 0) + + def test_live_hint_prompt_lookups_use_indexes(self): + db = self.database() + conn = db.model.get_connection() + plan = [ + row[3] + for row in conn.execute("EXPLAIN QUERY PLAN " + LIVE_HINT_PROMPT_IDS_SQL) + ] + prompt_steps = [ + step for step in plan if " p " in f" {step} " or "TABLE p" in step + ] + self.assertTrue(prompt_steps, plan) + self.assertEqual([step for step in plan if step.startswith("SCAN p")], []) + self.assertEqual([step for step in plan if step.startswith("SCAN q")], []) + def test_hints_require_exact_unique_raw_text_and_never_refresh(self): db = self.database() conn = db.model.get_connection() diff --git a/tests/test_execution_history_hooks.py b/tests/test_execution_history_hooks.py index dbb03655..2f264d9b 100644 --- a/tests/test_execution_history_hooks.py +++ b/tests/test_execution_history_hooks.py @@ -19,6 +19,7 @@ from unittest.mock import patch from tests.history_waits import drain_writer from database import PromptDatabase from utils import usage_tracking +from utils.hashing import generate_prompt_hash from utils.execution_history_writer import HistoryWriter try: @@ -918,6 +919,30 @@ class CachedRuntimeHistoryTests(HooksCase): self.assertIsNone(named_prompt(self.db, self.rows()[1]["run_id"])) +class CachedRuntimeCaseHistoryTests(CachedRuntimeHistoryTests): + """Raw text that differs only in case joins the saved prompt's history (#460). + + The hash folds case, so the run counts toward the lower-case prompt saved + first; the run keeps its exact raw spelling. + """ + + TEXT = "\u00c9 LIGHTHOUSE the LLM wrote" + + def setUp(self): + super().setUp() + self.db.delete_prompt(self.prompt_id) + lower = self.TEXT.lower() + self.prompt_id = self.db.save_prompt( + text=lower, prompt_hash=generate_prompt_hash(lower) + ) + + def test_the_library_counts_the_case_variant_as_this_prompt(self): + self.assertEqual( + self.db.get_prompt_by_hash(generate_prompt_hash(self.TEXT))["id"], + self.prompt_id, + ) + + class CachedRuntimeWhitespaceHistoryTests(CachedRuntimeHistoryTests): """Raw text with outer whitespace joins the history of the prompt it saves (#438). diff --git a/tests/test_hashing.py b/tests/test_hashing.py index a06eec86..0f4d41a2 100644 --- a/tests/test_hashing.py +++ b/tests/test_hashing.py @@ -17,6 +17,8 @@ from utils.hashing import ( generate_prompt_hash, is_duplicate_prompt, STRIPPED_CODE_POINTS, + prompt_key, + register_prompt_functions, stored_prompt_text, stored_prompt_text_sql, ) @@ -158,5 +160,56 @@ class TestStoredPromptText(unittest.TestCase): self.assertEqual(self.sql_strip(value), stored_prompt_text(value)) +class TestPromptKey(unittest.TestCase): + """History compares the hash's own key; SQL calls Python for it (#460).""" + + CASES = ( + "A Lighthouse", + " \u00c9t\u00c9\n", + "\u03a3\u03a3", + "\u0130stanbul", + "\u1e9e", + "\u212a", + "ok", + ) + + def test_key_is_the_hash_normalization(self): + import hashlib + + for value in self.CASES: + with self.subTest(value=repr(value)): + self.assertEqual( + generate_prompt_hash(value), + hashlib.sha256(prompt_key(value).encode("utf-8")).hexdigest(), + ) + self.assertEqual(prompt_key(" A Lighthouse\n"), "a lighthouse") + + def test_sql_functions_give_pythons_result_where_sql_lower_does_not(self): + import sqlite3 + + conn = sqlite3.connect(":memory:") + try: + register_prompt_functions(conn) + for value in self.CASES: + with self.subTest(value=repr(value)): + key, digest = conn.execute( + "SELECT pm_prompt_key(?), pm_prompt_hash(?)", (value, value) + ).fetchone() + self.assertEqual(key, prompt_key(value)) + self.assertEqual(digest, generate_prompt_hash(value)) + self.assertEqual( + conn.execute( + "SELECT pm_prompt_key(1), pm_prompt_hash(NULL)" + ).fetchone(), + (None, None), + ) + # SQLite's own lower() folds ASCII only + self.assertNotEqual( + conn.execute("SELECT lower(?)", ("\u00c9",)).fetchone()[0], "\u00e9" + ) + finally: + conn.close() + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_history_reads.py b/tests/test_history_reads.py index c02ea1e8..73994f3c 100644 --- a/tests/test_history_reads.py +++ b/tests/test_history_reads.py @@ -17,6 +17,7 @@ from utils.execution_contexts import ( ) from utils.execution_outputs import OutputBundle from tests.test_execution_context_database import bundle, snapshot +from utils.hashing import generate_prompt_hash from tests.test_execution_output_database import bundle as outputs from tests import test_execution_history_database as history_tests @@ -33,6 +34,10 @@ class HistoryReadTests(unittest.TestCase): conn.execute("INSERT INTO prompts(id,text) VALUES(?,?)", (identity, text)) return identity + def hashed(self, db, text): + """A prompt saved as the nodes save it: with its dedup hash.""" + return db.save_prompt(text, prompt_hash=generate_prompt_hash(text)) + def capture(self, db, item=0, context=None, output=None, time=100): start = self.start(db, item, time) db.write_execution_start(start) @@ -151,9 +156,9 @@ class HistoryReadTests(unittest.TestCase): ) self.require_reads(db) with db.recorded_history_snapshot(): - # Outer whitespace joins the stored form's scope (#438); case and - # Unicode normalization stay distinct. - expected = (4, 1, 4, 4, 1, 1, 2, 2) + # Outer whitespace (#438) and case (#460) join one scope; + # Unicode normalization stays distinct. + expected = (5, 5, 5, 5, 1, 1, 2, 2) for i, count in enumerate(expected, 1): self.assertEqual(db.list_history_window(i)["recorded_run_count"], count) self.assertIsNone(db.list_history_window(100)) @@ -1047,3 +1052,70 @@ class HistoryReadTests(unittest.TestCase): with db.recorded_history_snapshot(): found = db.get_newest_pair_run(positive, pair) self.assertEqual(found["run"]["run_id"], newest.run_id) + + def test_case_variant_raw_text_joins_the_saved_prompt_scope(self): + # #460: the hash folds case, so these runs count toward this prompt. + db = self.database() + positive = self.hashed(db, "\u00e9t\u00e9 lighthouse \u03c3") + raws = ( + "\u00c9T\u00c9 Lighthouse \u03a3\n", + "\u00e9t\u00e9 lighthouse \u03c3", + " \u00c9t\u00e9 LIGHTHOUSE \u03a3", + ) + starts = [ + self.capture(db, i, bundle(positive=raw), time=100 + i) + for i, raw in enumerate(raws) + ] + # Unicode normalization and inner text stay distinct + self.capture(db, 7, bundle(positive="e\u0301te\u0301 lighthouse \u03c3")) + self.capture(db, 8, bundle(positive="\u00e9t\u00e9 lighthouse \u03c3")) + dotted = self.hashed(db, "\u0130stanbul") + dotted_run = self.capture(db, 9, bundle(positive="\u0130STANBUL"), time=300) + with db.recorded_history_snapshot(): + page = db.list_history_window(positive) + details = [db.get_history_run(start.run_id) for start in starts] + turkish = db.list_history_window(dotted) + self.assertEqual(page["recorded_run_count"], 3) + self.assertEqual( + [r["run_id"] for r in page["items"]], [s.run_id for s in starts] + ) + for raw, detail in zip(raws, details): + context = detail["contexts"][0] + self.assertEqual(context["positive"]["raw"], raw) + self.assertEqual( + context["positive_raw_text_id"], page["group"]["raw_text_id"] + ) + self.assertEqual(context["positive_prompt_id"], positive) + self.assertEqual([r["run_id"] for r in turkish["items"]], [dotted_run.run_id]) + + def test_newest_pair_run_follows_case_variant_raw_text(self): + db = self.database() + pair = db.save_prompt_pair("a lighthouse", "fog") + positive = db.get_prompt_pair(pair)["positive_prompt_id"] + newest = self.capture(db, 0, bundle("FOG", positive="A Lighthouse"), time=100) + with db.recorded_history_snapshot(): + found = db.get_newest_pair_run(positive, pair) + self.assertEqual(found["run"]["run_id"], newest.run_id) + + def test_a_position_from_before_460_resumes_the_same_scope(self): + db = self.database() + positive = self.hashed(db, "A Lighthouse") + for i in range(3): + self.capture(db, i, bundle(positive="a lighthouse"), time=100 + i) + with db.recorded_history_snapshot(): + first = db.list_history_window(positive, limit=2) + position = first["next_position"] + legacy = replace( + position, raw_text_id=content_digest("text", ("A Lighthouse",)) + ) + rest = db.list_history_window(positive, limit=2, after=legacy) + with self.assertRaises(ValueError): + db.list_history_window( + positive, + after=replace(position, raw_text_id=content_digest("text", ("x",))), + ) + self.assertEqual(first["recorded_run_count"], 3) + self.assertEqual(len(rest["items"]), 1) + self.assertEqual( + rest["group"]["raw_text_id"], content_digest("text", ("a lighthouse",)) + ) diff --git a/utils/hashing.py b/utils/hashing.py index 65582f8e..ccf714e4 100644 --- a/utils/hashing.py +++ b/utils/hashing.py @@ -58,11 +58,7 @@ def generate_prompt_hash(text: str) -> str: if not isinstance(text, str): raise TypeError("Text must be a string") - # Normalize the text by stripping whitespace and converting to lowercase - # for consistent hashing regardless of minor formatting differences - normalized_text = text.strip().lower() - - return hashlib.sha256(normalized_text.encode("utf-8")).hexdigest() + return hashlib.sha256(prompt_key(text).encode("utf-8")).hexdigest() def stored_prompt_text(text: str) -> str: @@ -93,6 +89,34 @@ STRIPPED_CODE_POINTS = ( _STRIP_CODES = ",".join(map(str, STRIPPED_CODE_POINTS)) +def prompt_key(text: str) -> str: + """The dedup key: the stored text with case folded by ``str.lower()``. + + Two texts with one key are one prompt (the hash is sha256 of the key), so + a run belongs to the history of the prompt with its key (#460). + """ + return stored_prompt_text(text).lower() + + +def _sql_key(value): + return prompt_key(value) if isinstance(value, str) else None + + +def _sql_hash(value): + return generate_prompt_hash(value) if isinstance(value, str) else None + + +def register_prompt_functions(conn) -> None: + """Give ``conn`` ``pm_prompt_key(x)`` and ``pm_prompt_hash(x)`` (#460). + + SQLite's lower() folds ASCII only; these call Python, so SQL gets exactly + ``prompt_key`` / ``generate_prompt_hash`` (NULL for a non-text value). + Never used in schema objects, so the file stays readable without them. + """ + conn.create_function("pm_prompt_key", 1, _sql_key, deterministic=True) + conn.create_function("pm_prompt_hash", 1, _sql_hash, deterministic=True) + + def stored_prompt_text_sql(expression: str) -> str: """``stored_prompt_text`` as an SQLite expression over ``expression``. From 652458e995988ac86319b2f7163b92850c052cc2 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Thu, 29 Oct 2026 14:06:42 +0000 Subject: [PATCH 6/7] fix(history): send the file's image tags with History's current-file read (#457) (#462) History's current-file inspection now carries the same image_tags facet as the Images read, so History's inline current file and its viewer show the "Image tags" section added in #449. The facet reader moves to inspection_service.image_tags_reader and both routes use it; History keys it by the canonical target, the locator the Tag dialog stores. Closes #457 --- TESTING.md | 7 +- py/api/history.py | 14 ++- py/api/images.py | 24 +---- py/api/inspection_service.py | 29 ++++++ tests/browser/workspace-history.spec.mjs | 36 +++++-- tests/test_api_history_outputs.py | 116 +++++++++++++++++++++++ tests/test_inspection_response.py | 30 ++++++ utils/inspection_response.py | 7 +- 8 files changed, 227 insertions(+), 36 deletions(-) diff --git a/TESTING.md b/TESTING.md index 89aa2e40..1fb6a93b 100644 --- a/TESTING.md +++ b/TESTING.md @@ -1830,8 +1830,9 @@ Inspect plan keeps D60–D66 (plan D60 → D83, D61 → D84, D65 → D88, D68 blocks' context ("Current file image tags" in History). An empty source draws no section. Repeats are removed within a source only, so a linked tag that is also an image tag shows in both sections and each count is true. D21's filter rules are - unchanged. History's current-file read carries no `image_tags` facet (D88), so there - only the linked section shows. `tests/browser/inspector-concept.spec.mjs` wraps the + unchanged. History's current-file read carries the same `image_tags` facet as the Images + read (#457), so History's inline current file and its viewer show the image tags section + too (`test_current_inspection_carries_the_files_image_tags`). `tests/browser/inspector-concept.spec.mjs` wraps the concept's row in the same sections at runtime (`withTagSections`, the D154 rule injected) and compares the measured sections (frame, heading, label, row); a mutation of the label color, the row margin or the top rule fails it. Browser tests: @@ -6332,7 +6333,7 @@ The versioned public reads are: | `GET /prompt_manager/library/{positive_id}/history/pairs/{pair_id}/newest` | The pair's newest recorded run for the history run dot (#249 D26): `{run: {run_id, anchor_id, context_id}}` or `{run: null}` | | `GET /prompt_manager/history/runs/{run_id}` | Frozen fields, per-field provenance, capture and outcome for a known UUID | | `GET /prompt_manager/history/runs/{run_id}/outputs` | Ordered host-reported occurrences, including repeated locations | -| `GET /prompt_manager/history/runs/{run_id}/output/current/{media,thumbnail,inspection}` | Authorized current file at that reported location, using the returned epoch/scope URL | +| `GET /prompt_manager/history/runs/{run_id}/output/current/{media,thumbnail,inspection}` | Authorized current file at that reported location, using the returned epoch/scope URL; `inspection` carries the shared inspector envelope including the file's `image_tags` facet (#457) | The newest-pair read takes no query. It answers 404 when the positive is not an eligible saved prompt or the pair is not one of its pairs. It walks the positive's exact diff --git a/py/api/history.py b/py/api/history.py index 9426ee0c..a62c9821 100644 --- a/py/api/history.py +++ b/py/api/history.py @@ -27,7 +27,11 @@ from .history_current import ( resolve_current_candidate, snapshot_current_output_scope, ) -from .inspection_service import CurrentFileChanged, inspect_current_file +from .inspection_service import ( + CurrentFileChanged, + image_tags_reader, + inspect_current_file, +) from .images import image_category_key try: @@ -778,6 +782,14 @@ class HistoryRoutesMixin: ), own, ), + # The file's own image tags, keyed as the Tag dialog + # keys them: the normcased realpath (#457). + image_tags=image_tags_reader( + db, + str(resolved.canonical_target), + self.logger, + close=True, + ), recorded=recorded_output_lookup( db, close=True, reference=receipt["reference"] ), diff --git a/py/api/images.py b/py/api/images.py index 4726d995..01b27e3f 100644 --- a/py/api/images.py +++ b/py/api/images.py @@ -55,6 +55,7 @@ from .inspection_service import ( IMAGE_EXTENSIONS as _IMAGE_EXTENSIONS, SERVABLE_EXTENSIONS, VIDEO_EXTENSIONS, + image_tags_reader, inspect_current_file, ) @@ -3962,28 +3963,9 @@ class ImageRoutesMixin: ) def _inspection_image_tags(self, root: Path, parts: tuple) -> Callable: - """The inspector's ``image_tags`` reader for one rooted file (S1). - - The returned function takes the inspection's own stable stat and - answers ``current`` only for a byte record verified at exactly that - ``(size, mtime_ns)``; a failed read is ``unavailable``, never an error. - """ + """The inspector's ``image_tags`` reader for one rooted file (S1).""" locator = os.path.normcase(os.path.join(os.path.realpath(root), *parts)) - - def read(size: int, mtime_ns: int) -> dict: - try: - found = self.db.current_image_tags({locator: (size, mtime_ns)}) - except Exception: - self.logger.warning( - "Inspection image tags could not be read", exc_info=True - ) - return {"state": "unavailable", "tags": []} - tags = found.get(locator) - if tags is None: - return {"state": "none", "tags": []} - return {"state": "current", "tags": list(tags)} - - return read + return image_tags_reader(self.db, locator, self.logger) def _read_own_category(self, sources, path): """One file's own category, None, or OWN_CATEGORY_UNREAD (blocking). diff --git a/py/api/inspection_service.py b/py/api/inspection_service.py index 3a99af86..b635d889 100644 --- a/py/api/inspection_service.py +++ b/py/api/inspection_service.py @@ -93,6 +93,35 @@ def _in_host_temp(path: Path) -> bool: return False +def image_tags_reader( + db: Any, locator: str, logger: Any, *, close: bool = False +) -> Callable[[int, int], dict]: + """The inspector's ``image_tags`` reader for one file *locator* (S1). + + The returned function takes the inspection's own stable stat and answers + ``current`` only for a byte record verified at exactly that + ``(size, mtime_ns)``; a failed read is ``unavailable``, never an error. + ``close`` closes the calling thread's connection afterwards, for routes + that close theirs after reading (History). + """ + + def read(size: int, mtime_ns: int) -> dict: + try: + found = db.current_image_tags({locator: (size, mtime_ns)}) + except Exception: + logger.warning("Inspection image tags could not be read", exc_info=True) + return {"state": "unavailable", "tags": []} + finally: + if close: + db.model.close() + tags = found.get(locator) + if tags is None: + return {"state": "none", "tags": []} + return {"state": "current", "tags": list(tags)} + + return read + + def inspect_current_file( path: Path, *, diff --git a/tests/browser/workspace-history.spec.mjs b/tests/browser/workspace-history.spec.mjs index b8ae2588..d24e391b 100644 --- a/tests/browser/workspace-history.spec.mjs +++ b/tests/browser/workspace-history.spec.mjs @@ -1,6 +1,7 @@ import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; import {test,expect} from './fixtures.mjs'; import {armUnseenRestores,unseenRestore} from './unseen-restore.mjs'; +import {saveImageTags} from './concept-inventory.mjs'; import {readFile,writeFile} from 'node:fs/promises'; import {createHash} from 'node:crypto'; @@ -495,23 +496,38 @@ test('the inline current file names its prompts apart from the frozen run fields expect(await exact(dialog(page).getByRole('region',{name:'Positive prompt',exact:true}).locator('p.prompt-text'))).toBe(before); }); -// #449 (D154): History's inline current file draws tags in the shared labelled sections, named with its context so they -// stay apart from the frozen run's fields. The server sends History no image_tags facet today (D88), so the read is -// given one here to prove the shared render in History. -test('the inline current file labels its image tags after its negative prompt',async({page})=>{ +// #449 (D154), #457: History's inline current file draws the file's own image tags in the shared labelled section, named +// with its context so they stay apart from the frozen run's fields. The tags are saved on the real file through the Tag +// routes and come from History's own current-file read (its `image_tags` facet), as Images reads them. +test('the inline current file labels its image tags after its negative prompt',async({page,request})=>{ + const reads=[]; await page.route(url=>url.pathname.endsWith('/inspection'),async route=>{const response=await route.fetch();const body=await response.json(); + reads.push(body.image_tags); body.prompts={positive:{status:'resolved',text:'current file positive',source:'graph-inferred'},negative:{status:'resolved',text:'current file negative',source:'graph-inferred'}}; - body.image_tags={state:'current',tags:['history_tag']}; await route.fulfill({response,json:body});}); - await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5); - await dialog(page).getByRole('button',{name:'Raw',exact:true}).click(); - await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click(); - await expect(page.locator('dialog#viewer-dialog')).toBeVisible();await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible(); const current=dialog(page).getByRole('region',{name:'Current reported file',exact:true}); const tags=current.getByRole('region',{name:'Current file image tags',exact:true}); - await expect(tags.locator('.role-label')).toHaveText('Image tags · 1');await expect(tags.locator('.tags > .chip')).toHaveText(['history_tag']); + const showCurrent=async()=>{ + await open(page,page.fixture.detail_id);await marker(page,page.fixture.runs.shared).click();await expect(dialog(page).locator('[data-receipt]')).toHaveCount(5); + await dialog(page).getByRole('button',{name:'Raw',exact:true}).click(); + await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click(); + await expect(page.locator('dialog#viewer-dialog')).toBeVisible();await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible(); + await expect(current.getByRole('region',{name:'Current file negative prompt',exact:true})).toBeVisible(); + }; + // An untagged file draws no image tags section. + await showCurrent();await expect(tags).toHaveCount(0);expect(reads.at(-1)).toEqual({state:'none',tags:[]}); + await dialog(page).getByRole('button',{name:'Close recorded run',exact:true}).click(); + expect(await saveImageTags(request,'history/current.png',['history_tag','dusk'])).toEqual(['history_tag','dusk']); + await showCurrent();expect(reads.at(-1)).toEqual({state:'current',tags:['history_tag','dusk']}); + await expect(tags.locator('.role-label')).toHaveText('Image tags · 2');await expect(tags.locator('.tags > .chip')).toHaveText(['history_tag','dusk']); const negative=current.getByRole('region',{name:'Current file negative prompt',exact:true}); expect(await negative.evaluate((node,section)=>({after:Boolean(node.compareDocumentPosition(section)&Node.DOCUMENT_POSITION_FOLLOWING),inside:node.contains(section)}),await tags.elementHandle())).toEqual({after:true,inside:false}); + // The viewer opened from History shares the inspection: the same section, plain chips (no Tags filter there, D88). + await dialog(page).locator('[data-receipt]').first().getByRole('button',{name:'View current file',exact:true}).click(); + const viewerTags=page.locator('dialog#viewer-dialog #viewer-details section.tag-section.image-tags'); + await expect(viewerTags.locator('.role-label')).toHaveText('Image tags · 2'); + await expect(viewerTags.locator('span.chip')).toHaveText(['history_tag','dusk']);await expect(viewerTags.locator('button')).toHaveCount(0); + await page.keyboard.press('Escape');await expect(dialog(page)).toBeVisible(); }); // #332 CUJ 5 (MC8, §5.3): History's current-file inspector reads the file's own category with the Images key rule, so diff --git a/tests/test_api_history_outputs.py b/tests/test_api_history_outputs.py index c43f43f1..b3832030 100644 --- a/tests/test_api_history_outputs.py +++ b/tests/test_api_history_outputs.py @@ -1015,6 +1015,122 @@ class HistoryOutputAPITests(ImageAPITestCase): response = await self.current(descriptor["inspection_url"]) self.assert_error(response, await response.json(), 422, "response-limit") + async def tag_image(self, relative, tags): + """Save *tags* on Images root 0's *relative* file through the Tag routes. + + The real observe and save routes store the locator the Tag dialog + stores (``media_capture._resolve``), so History must find it (#457). + """ + self.addCleanup(self.api.close_annotation_captures) + roots = self.api._get_all_output_dirs() + target = { + "root_index": 0, + "root_scope": self.api._gallery_root_scope(roots[0], 0), + "path": relative, + } + response = await self.client.post( + "/prompt_manager/image-annotations/observe", json={"target": target} + ) + observed = await response.json() + self.assertEqual(response.status, 200, observed) + token = observed["capture"]["token"] + try: + response = await self.client.put( + "/prompt_manager/image-annotations/tags", + json={ + "target": target, + "capture_token": token, + "sha256": observed["capture"]["sha256"], + "expected_epoch": observed["library_epoch"], + "expected_tag_revision": observed["tag_revision"], + "tags": list(tags), + }, + ) + saved = await response.json() + self.assertEqual(response.status, 200, saved) + finally: + await self.client.delete( + f"/prompt_manager/image-annotations/capture/{token}" + ) + return saved["tags"] + + async def test_current_inspection_carries_the_files_image_tags(self): + """#457: History reads the file's own image tags, as Images does (S1).""" + sub = self.output_dir / "batch" + sub.mkdir() + tagged = png(sub / "tagged.png", prompt=graph("Tagged file")) + png(self.output_dir / "plain.png", prompt=graph("Plain file")) + self.assertEqual( + await self.tag_image("batch/tagged.png", ["dunes", "warm_light"]), + ["dunes", "warm_light"], + ) + start = self.capture( + [ + self.ref("1", subfolder="batch", filename="tagged.png"), + self.ref("2", filename="plain.png"), + ] + ) + _, page = await self.outputs(start) + urls = [ + item["current_candidate"]["media"]["inspection_url"] + for item in page["items"] + ] + bodies = [] + for url in urls: + response = await self.current(url) + body = await response.json() + self.assertEqual(response.status, 200, body) + bodies.append(body) + self.assertEqual( + bodies[0].get("image_tags"), + {"state": "current", "tags": ["dunes", "warm_light"]}, + ) + # Image tags are the file's own search metadata, never prompt tags. + self.assertEqual(bodies[0]["prompts"]["positive"]["text"], "Tagged file") + self.assertEqual(bodies[1].get("image_tags"), {"state": "none", "tags": []}) + # New bytes at the same path: the tags belong to the old byte record. + stat = tagged.stat() + os.utime(tagged, ns=(stat.st_atime_ns, stat.st_mtime_ns + 10**9)) + response = await self.current(urls[0]) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual(body["image_tags"], {"state": "none", "tags": []}) + + async def test_current_inspection_finds_image_tags_under_a_symlinked_output(self): + """An output folder reached through a link (another drive) still matches.""" + link = self.comfy_dir / "linked_output" + self._make_symlink(self.output_dir, link, target_is_directory=True) + self.api._get_all_output_dirs = lambda: [link] + self.host_output = str(link) + png(self.output_dir / "image.png", prompt=graph("Linked folder")) + self.assertEqual(await self.tag_image("image.png", ["dunes"]), ["dunes"]) + start = self.capture([self.ref()]) + descriptor = await self.media(start) + response = await self.current(descriptor["inspection_url"]) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual(body["image_tags"], {"state": "current", "tags": ["dunes"]}) + + async def test_current_inspection_image_tag_failure_is_unavailable(self): + png(self.output_dir / "image.png", prompt=graph("File evidence")) + await self.tag_image("image.png", ["dunes"]) + start = self.capture([self.ref()]) + descriptor = await self.media(start) + with ( + self.assertLogs(self.api.logger, "WARNING") as logs, + patch.object( + type(self.api.db), + "current_image_tags", + side_effect=sqlite3.OperationalError("locked"), + ), + ): + response = await self.current(descriptor["inspection_url"]) + body = await response.json() + self.assertEqual(response.status, 200, body) + self.assertEqual(body["image_tags"], {"state": "unavailable", "tags": []}) + self.assertEqual(body["prompts"]["positive"]["text"], "File evidence") + self.assertNotIn(str(self.output_dir), "\n".join(logs.output)) + async def test_optional_candidate_failure_preserves_every_occurrence_without_urls( self, ): diff --git a/tests/test_inspection_response.py b/tests/test_inspection_response.py index 6b8412be..6188ce2e 100644 --- a/tests/test_inspection_response.py +++ b/tests/test_inspection_response.py @@ -887,6 +887,36 @@ class TestImageTagsFacet(unittest.TestCase): self.assertNotIn("image_tags", json.loads(encode(inspection({})))) self.assertNotIn("image_tags", self.encoded(None)) + def history_encoded(self, facet): + from utils.inspection_response import encode_history_inspection_response + + body = encode_history_inspection_response( + current_output={"history_epoch": "epoch"}, + file={"observation": "current-file", "size": 100}, + prompts=PROMPTS, + resolution={"state": "resolved", "sampler_id": "3", "reason": ""}, + inspection=inspection({"prompt": graph_text()}), + library={"status": "none"}, + image_tags=facet, + ) + return json.loads(body) + + def test_history_envelope_validates_image_tags_like_the_rooted_one(self): + """#457: History's current-file read carries the same facet.""" + for facet in ( + {"state": "current", "tags": ["dunes", "warm_light"]}, + {"state": "none", "tags": []}, + {"state": "unavailable", "tags": []}, + {"state": "current", "tags": ["a,b"]}, + {"state": "current", "tags": [], "locator": "/abs/path.png"}, + ): + with self.subTest(facet=facet): + self.assertEqual( + self.history_encoded(facet)["image_tags"], + self.encoded(facet)["image_tags"], + ) + self.assertNotIn("image_tags", self.history_encoded(None)) + class TestVideoProbeBoundsMatchPngBudgets(unittest.TestCase): """ffprobe escapes tag text; the stdout bounds must never gate a record diff --git a/utils/inspection_response.py b/utils/inspection_response.py index ba6a43f0..4aff7885 100644 --- a/utils/inspection_response.py +++ b/utils/inspection_response.py @@ -548,8 +548,12 @@ def encode_history_inspection_response( resolution: dict, inspection: "InspectionRead | None", library: dict, + image_tags: dict | None = None, ) -> bytes: - """Use occurrence identity with the same current-file evidence projection.""" + """Use occurrence identity with the same current-file evidence projection. + + *image_tags*, when given, adds the validated ``image_tags`` facet (#457). + """ return _encode_inspection( identity={ "history_version": 1, @@ -561,4 +565,5 @@ def encode_history_inspection_response( resolution=resolution, inspection=inspection, library=library, + image_tags=image_tags, ) From 012670504591f10be7d82ecd0ec3e3fad0f60cf7 Mon Sep 17 00:00:00 2001 From: Vito <vito.sansevero@gmail.com> Date: Thu, 29 Oct 2026 14:10:29 +0000 Subject: [PATCH 7/7] feat(library): filter prompts by output folder and browse tags with Any (#444, PR 2) (#456) * feat(vocabulary): tag search, tag sort and an Untagged prompts entry (#444) Tags & categories gains a server-side tag search and sort (keyset pages, cursor scoped to the query) and an "Untagged prompts" row whose Browse opens the Library with tag_mode=untagged. Refine this view gets Tag match, saved Library views keep it, and the routes echo the applied query so a stale server is an error, never an unfiltered list. Recorded as D149. Refs #444 * feat(library): filter prompts by output folder and browse tags with Any (#444) Refine this view on the Prompt library gains the concept's Folder (folders below an output root, read when the dialog opens) and a Tag match select (All / Any). The listing takes folder and tag_mode, matches folders on the normalized file_path under every spelling of each output root, echoes both so a stale server is an error, and saved Library views keep them. Recorded as D150. Refs #444 * fix(library): let saved-view defaults fill the missing #444 filters libraryViewState passed `?? 'all'` and `?? ''` fallbacks that normalizeViewState already applies to an undefined value. The redundant branches were never taken in the page, which dropped saved-views.mjs below its 95% branch floor. A unit now covers a shelf without the new filters. Refs #444 * test(saved-views): give the D153 Library view the tag_mode key (#444) The Library saved-view state now holds tag_mode, and the API requires every key, so main's new "Highest rated" spec posts it too. Refs #444 * test(saved-views): give the D153 Library view the folder key (#444) Refs #444 * fix(library): nest Refine folders by path segment; backslash separates only on Windows (#444) Live test B1: the Folder list indented a server list sorted as full-path text, so "-" and "_" sorted before "/" and children landed under the wrong parent (kiko/char under kiko-v6), with case variants interleaved. The options are now a tree built by segment: each folder directly under its own parent, case variants distinct siblings. On POSIX a folder named "Metadata\Klein..." was listed but filtered to no prompts, because normalize_folder read "\" as a separator. It now does so only on Windows (ntpath), as the listing already did. Refs #444 --- TESTING.md | 46 +++- database/library.py | 75 ++++-- database/library_folders.py | 114 ++++++++ database/saved_views.py | 8 +- py/api/library.py | 53 +++- .../workspace-library-filters.spec.mjs | 143 +++++++++++ tests/browser/workspace-library.spec.mjs | 5 +- tests/browser/workspace-saved-views.spec.mjs | 12 +- tests/js/workspace-library-state.test.js | 41 ++- tests/js/workspace-saved-views.test.js | 29 ++- tests/test_api_library.py | 66 +++++ tests/test_library_folders.py | 243 ++++++++++++++++++ tests/test_saved_views.py | 34 ++- web/js/workspace/library-state.mjs | 37 ++- web/js/workspace/library.mjs | 47 ++-- web/js/workspace/saved-views.mjs | 10 +- web/js/workspace/workspace-dialogs.mjs | 2 +- 17 files changed, 875 insertions(+), 90 deletions(-) create mode 100644 database/library_folders.py create mode 100644 tests/browser/workspace-library-filters.spec.mjs create mode 100644 tests/test_library_folders.py diff --git a/TESTING.md b/TESTING.md index 1fb6a93b..7b1cb16d 100644 --- a/TESTING.md +++ b/TESTING.md @@ -2443,10 +2443,38 @@ Recorded deviations on this page: view" filters are active, and plain "Filters" otherwise; the text is its accessible name, so the count is spoken. The concept has no indicator, and without one a Tags filter set by Tags & categories Browse was invisible on the shelf. The Library counts - Tags; Images counts Source, Folder, Media and Tags (each once). The toolbar category is + Tags, Folder and Tag match (D150); Images counts Source, Folder, Media and Tags (each once). The toolbar category is not a Refine filter and is not counted. The caption's count is a `role="status"` region, written only when it changes and kept shown while the shelf reloads, so Load more and scrolling never re-announce it. +- **D150** (#444, cutover #334): "Refine this view" on the Library holds the concept + Filters dialog's **Folder** ("All output folders", then each folder below an output + root as a tree built by path segment (each folder directly under its own parent, + case variants as distinct siblings, siblings case-insensitive then exact), indented + by depth, read from `GET /prompt_manager/library/options?include=folders` + when the dialog opens; the chosen folder stays listed even when the list lacks it) and + **Tags**, plus **Tag match** ("All of these tags" / "Any of these tags" / "Prompts without tags", D149) drawn as the + concept's own select; the concept's "Prompt evidence" and prototype note are absent. + `GET /prompt_manager/library` takes `folder` (`/`-joined; `\` is a separator only on + Windows, on POSIX it is part of a folder's name as the listing shows it; `.`, `..`, + empty segments and, on Windows, drives are 400) and `tag_mode` (`all` default, `any`, `untagged` from D149), and echoes + both; a client whose non-default value is not echoed (an un-restarted older server) + shows the read error, never the unfiltered shelf. A folder keeps a parent with one + linked image (`generated_images.prompt_id`) in it or below it under any spelling of + any output root (the root as configured or detected and after `realpath`, so a + symlinked output folder works): `file_path` (`normcase(normpath(abspath))`) is + compared with prefixes built by the same normalization, so case folding is the + platform's own, including non-ASCII on Windows. The folder is an outer AND around the + parent and pair arms; Any puts one `t.name IN (...)` in each arm, so a card matches + when the parent or one active pair carries any tag (text and category still AND + inside each arm); with one tag Any equals All. Filters counts Tags, Folder and a Tag + match other than All (D44). Saved Library views store `tag_mode` (`all|any|untagged`) and + `folder` (text 1024); older rows read the defaults. Tests: + `tests/test_library_folders.py` (pure prefixes and names with `posixpath`/`ntpath`, + drive, UNC, non-ASCII, symlinked root, the any-tag union), `test_api_library.py`, + `test_saved_views.py`, the library-state and saved-views JS units, and + `tests/browser/workspace-library-filters.spec.mjs` (behavior, the Refine dialog against + the concept Filters dialog, artifact captures). Windows: run `portability.yml` on the PR. `tests/browser/library-concept.spec.mjs` compares the shelf (#249 E3) on the concept-shaped fixture, which creates the concept's prompts last to first so Recently @@ -6637,12 +6665,12 @@ Deploying this backend needs a ComfyUI restart (Python and route changes). `#/tags` ("Tags & categories", page title "Tags") is a native workspace route; it no longer hands off to the legacy tag manager. That page stays reachable from Overview's -"Open existing tag manager" card for multi-tag AND/OR browsing (tag search and sort -and the untagged entry moved here with D149, #444); bulk prompt delete stays on the -legacy admin page (bulk tags: the Prompt library's selection bar, D151). A -prompt's category is changed in the Prompt library editor, on many prompts at once with -the Prompt library's selection bar (D148, #437), or with the legacy admin page's "Set -Category" (both `POST /prompt_manager/bulk/category`); +"Open existing tag manager" card for its own page; tag search and sort, the untagged +entry (D149) and multi-tag OR (the Library's Tag match, D150) moved to the workspace +(#444); bulk prompt delete stays on the legacy admin page (bulk tags: the Prompt +library's selection bar, D151). A prompt's category is changed in the Prompt library +editor, on many prompts at once with the Prompt library's selection bar (D148, #437), or +with the legacy admin page's "Set Category" (both `POST /prompt_manager/bulk/category`); the Images selection bar's Set category sets images' own categories only (D140, #332), and Manage category renames a value on prompts, recipes and images together (D144). The screen consumes only the `/prompt_manager/vocabulary/*` routes above and adds no route, so PR B needed no ComfyUI restart; D149 (#444) adds read @@ -6715,10 +6743,10 @@ fixture-only tags after the concept's sample tags. counted every prompt without `prompt_tags` and hid the entry at 0; this row always shows) and echoes `search` and `sort`. Browse opens the Library with `tag_mode=untagged` (`GET /prompt_manager/library`, echoed as `tag_mode`; with a `tag` it is 400): "Filters · - 1", and Refine this view adds **Tag match** after Tags ("All of these tags" / "Prompts + 1", and Refine this view adds **Tag match** after Tags ("All of these tags" / "Any of these tags" (D150) / "Prompts without tags", the latter disables Tags and drops them on Apply). A client whose request the server did not echo (an un-restarted older server) shows the read error with Retry, - never an unfiltered list. Saved Library views store `tag_mode` (`all|untagged`, default + never an unfiltered list. Saved Library views store `tag_mode` (`all|any|untagged`, default `all`); untagged with tags is refused on write and reads back as `all`; an older build opens an untagged view as the whole Library. The concept comparison holds the row to the concept's first category row, its top to the concept table's offset, the table to that diff --git a/database/library.py b/database/library.py index ec684527..49f18ac9 100644 --- a/database/library.py +++ b/database/library.py @@ -5,6 +5,7 @@ import sqlite3 from collections.abc import Iterator from contextlib import contextmanager +from .library_folders import folder_condition, list_library_folders from .library_tokens import library_edit_token _ELIGIBLE = """(EXISTS(SELECT 1 FROM prompt_pairs ep WHERE ep.positive_prompt_id=p.id) @@ -121,6 +122,9 @@ def _archive(value: str) -> str: return ARCHIVE_SCOPES[value] +# How the Tags filter matches (#444): every tag (AND), any of them (OR), or +# parents without any tag. +TAG_MODES = ("all", "any", "untagged") # A parent that no tag's Browse returns (#444 "Untagged prompts"): no tag of its # own and no active pair carrying one. The vocabulary's untagged count uses it too. UNTAGGED = ( @@ -128,8 +132,6 @@ UNTAGGED = ( "AND NOT EXISTS(SELECT 1 FROM prompt_pairs up JOIN prompt_pair_tags upt " "ON upt.pair_id=up.id WHERE up.positive_prompt_id=p.id AND up.archived=0)" ) -# How the Tags filter matches: every tag (AND), or parents without any tag. -TAG_MODES = ("all", "untagged") def _tag_mode(tag_mode: object, tags: list[str]) -> str: @@ -140,8 +142,33 @@ def _tag_mode(tag_mode: object, tags: list[str]) -> str: return tag_mode +def _tag_conditions(tags: list[str], any_tag: bool) -> tuple[list, list, list]: + """Parent-arm and pair-arm tag conditions and the tags' parameters.""" + parent_exists = ( + "EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id " + "WHERE pt.prompt_id=p.id AND t.name{test})" + ) + pair_exists = ( + "EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id " + "WHERE pt.pair_id=pp.id AND t.name{test})" + ) + if any_tag and tags: + # One condition per arm: the card, or one active pair, carries any tag. + test = " IN (" + ",".join("?" for _ in tags) + ")" + return [parent_exists.format(test=test)], [pair_exists.format(test=test)], tags + return ( + [parent_exists.format(test="=?") for _ in tags], + [pair_exists.format(test="=?") for _ in tags], + tags, + ) + + def _filter( - text: str, category: str | None, tags: list[str], tag_mode: str = "all" + text: str, + category: str | None, + tags: list[str], + tag_mode: str = "all", + folder_prefixes: list[str] | None = None, ) -> tuple[str, list]: if not isinstance(text, str) or len(text) > 500: raise ValueError("Search text must be a string of at most 500 characters") @@ -153,7 +180,8 @@ def _filter( or any(not isinstance(tag, str) or not tag or len(tag) > 200 for tag in tags) ): raise ValueError("Invalid library tags") - untagged = _tag_mode(tag_mode, tags) == "untagged" + mode = _tag_mode(tag_mode, tags) + untagged = mode == "untagged" parent, pair, parent_params, pair_params = [], ["pp.archived=0"], [], [] if text: parent.append("(p.text LIKE ? ESCAPE '\\' OR ls.title LIKE ? ESCAPE '\\')") @@ -168,17 +196,11 @@ def _filter( pair.append("pp.category=?") parent_params.append(category) pair_params.append(category) - for tag in tags: - parent.append( - "EXISTS(SELECT 1 FROM prompt_tags pt JOIN tags t ON t.id=pt.tag_id " - "WHERE pt.prompt_id=p.id AND t.name=?)" - ) - pair.append( - "EXISTS(SELECT 1 FROM prompt_pair_tags pt JOIN tags t ON t.id=pt.tag_id " - "WHERE pt.pair_id=pp.id AND t.name=?)" - ) - parent_params.append(tag) - pair_params.append(tag) + parent_tags, pair_tags, tag_params = _tag_conditions(tags, mode == "any") + parent.extend(parent_tags) + pair.extend(pair_tags) + parent_params.extend(tag_params) + pair_params.extend(tag_params) condition = ( "(" + (" AND ".join(parent) or "1=1") @@ -194,7 +216,13 @@ def _filter( # A whole-card condition: inside the parent-or-pair OR an untagged pair # would let a tagged parent through. condition = f"({condition}) AND {UNTAGGED}" - return f"({_ELIGIBLE}) AND ({condition})", [*parent_params, *pair_params] + params = [*parent_params, *pair_params] + if folder_prefixes: + # A whole-card condition around both arms: images link to the parent. + folder, folder_params = folder_condition(folder_prefixes) + condition = f"({condition}) AND {folder}" + params.extend(folder_params) + return f"({_ELIGIBLE}) AND ({condition})", params class LibraryOperations: @@ -278,6 +306,7 @@ class LibraryOperations: category: str | None = None, tags: list[str] | None = None, tag_mode: str = "all", + folder_prefixes: list[str] | None = None, sort: str = "newest", limit: int = 40, after: dict | None = None, @@ -285,7 +314,9 @@ class LibraryOperations: _limit(limit) if sort not in _ORDERS: raise ValueError("Unknown library sort order") - where, params = _filter(text, category, [] if tags is None else tags, tag_mode) + where, params = _filter( + text, category, [] if tags is None else tags, tag_mode, folder_prefixes + ) if after is not None: if ( not isinstance(after, dict) @@ -351,6 +382,7 @@ class LibraryOperations: category: str | None = None, tags: list[str] | None = None, tag_mode: str = "all", + folder_prefixes: list[str] | None = None, sort: str = "newest", limit: int = 40, after: dict | None = None, @@ -364,6 +396,7 @@ class LibraryOperations: category=category, tags=tags, tag_mode=tag_mode, + folder_prefixes=folder_prefixes, sort=sort, limit=limit, after=after, @@ -494,9 +527,13 @@ class LibraryOperations: "categories": [row["category"] for row in rows], } - def get_library_options(self) -> dict: + def get_library_options(self, folder_roots=None) -> dict: + """Categories; with *folder_roots* (root spellings) also ``folders``.""" with self.library_snapshot() as (conn, epoch, _ceiling): - return self._get_library_options(conn, epoch) + result = self._get_library_options(conn, epoch) + if folder_roots is not None: + result["folders"] = list_library_folders(conn, folder_roots, _ELIGIBLE) + return result def get_library_previews(self, positive_ids: list[int]) -> dict[int, list[dict]]: """Read at most two links per parent in one bounded page query.""" diff --git a/database/library_folders.py b/database/library_folders.py new file mode 100644 index 00000000..48d10f7d --- /dev/null +++ b/database/library_folders.py @@ -0,0 +1,114 @@ +"""Library output-folder filter (#444): folder names, root prefixes and listing. + +A folder is a ``/``-joined path relative to an output root, as Images' folder +filter and the legacy ``/prompt_manager/subfolders`` name them. A parent +matches a folder when one of its linked images (``generated_images.prompt_id``, +the link its previews and image count read) lies in that folder or below it, +under any spelling of any output root. + +Matching compares ``generated_images.file_path``, which is always +``normcase(normpath(abspath(path)))`` (``models.normalize_image_path``), with +prefixes built by the same normalization here, so case folding is the +platform's own on both sides (Windows folds non-ASCII too, unlike SQLite's +``lower()``) and the comparison is an exact ``substr`` with no wildcards. +""" + +import os +import sqlite3 + +_MAX_FOLDER = 1024 +_MAX_PREFIXES = 64 + + +def normalize_folder(value: object, pathmod=os.path) -> str: + """The folder as ``a/b``, outer slashes dropped; empty means no filter. + + ``/`` always separates. ``\\`` separates only on Windows (``ntpath``): on + POSIX it is an ordinary character of a folder's name, and the listing + (``relative_folder``) names such a folder with it, so both agree. + ``.``/``..``/empty segments, control characters and (on Windows) a drive + are refused: the value only ever names a folder below a root. + """ + if not isinstance(value, str) or len(value) > _MAX_FOLDER: + raise ValueError(f"folder must be text of at most {_MAX_FOLDER} characters") + windows = pathmod.sep == "\\" + folder = (value.replace("\\", "/") if windows else value).strip("/") + if not folder: + return "" + parts = folder.split("/") + if ( + any(part in ("", ".", "..") for part in parts) + or (windows and ":" in parts[0]) + or any(ord(char) < 32 or ord(char) == 127 for char in folder) + ): + raise ValueError("folder must be a folder below an output root") + return folder + + +def _base(root: object, pathmod) -> str: + return pathmod.normcase(pathmod.normpath(str(root))) + + +def folder_prefixes(roots, folder: str, pathmod=os.path) -> list[str]: + """One ``file_path`` prefix per root spelling for a normalized folder.""" + prefixes = [] + for root in roots: + joined = pathmod.join(_base(root, pathmod), *folder.split("/")) + prefix = pathmod.normcase(pathmod.normpath(joined)) + pathmod.sep + if prefix not in prefixes: + prefixes.append(prefix) + if not prefixes or len(prefixes) > _MAX_PREFIXES: + raise ValueError("No output folder to filter by") + return prefixes + + +def folder_condition(prefixes: list[str]) -> tuple[str, list]: + """A condition on alias ``p``: one linked image under any prefix.""" + tests = " OR ".join("substr(fg.file_path,1,?)=?" for _ in prefixes) + params = [value for prefix in prefixes for value in (len(prefix), prefix)] + return ( + "EXISTS(SELECT 1 FROM generated_images fg WHERE fg.prompt_id=p.id " + f"AND ({tests}))", + params, + ) + + +def relative_folder(image_path: str, roots, pathmod=os.path) -> str | None: + """The folder of *image_path* below the first root spelling holding it. + + The spelling of the stored path is kept (its case on Windows); ``None`` + for an image directly in a root or outside every root. + """ + directory = pathmod.normpath(pathmod.dirname(image_path)) + folded = pathmod.normcase(directory) + for root in roots: + base = _base(root, pathmod) + start = base if base.endswith(pathmod.sep) else base + pathmod.sep + if not folded.startswith(start): + continue + # normcase keeps the length for ordinary names; else show the folded form. + rest = (directory if len(directory) == len(folded) else folded)[len(start) :] + parts = rest.split(pathmod.sep) + if any(part in ("", ".", "..") for part in parts): + return None + return "/".join(parts) + return None + + +def list_library_folders( + conn: sqlite3.Connection, roots, eligible: str, pathmod=os.path +) -> list[str]: + """Every folder (and ancestor) holding an image of an eligible parent.""" + folders = set() + for (image_path,) in conn.execute( + "SELECT DISTINCT g.image_path FROM generated_images g " + "JOIN prompts p ON p.id=g.prompt_id WHERE " + eligible + ): + if not isinstance(image_path, str) or not image_path: + continue + folder = relative_folder(image_path, roots, pathmod) + if folder is None: + continue + parts = folder.split("/") + folders.update("/".join(parts[:index]) for index in range(1, len(parts) + 1)) + return sorted(folders, key=lambda value: (value.casefold(), value)) diff --git a/database/saved_views.py b/database/saved_views.py index 9fad284c..9d691f43 100644 --- a/database/saved_views.py +++ b/database/saved_views.py @@ -33,8 +33,9 @@ MAX_TAG_LENGTH = 50 _IMAGE_SORTS = ("newest", "oldest", "name", "name-desc", "rating") _LIBRARY_SORTS = ("newest", "oldest", "name", "rating") _MEDIA = ("all", "image", "video") -# How a Library view's Tags match (#444): every tag, or parents without tags. -_TAG_MODES = ("all", "untagged") +# How a Library view's Tags match (#444): every tag, any of them, or parents +# without tags ("untagged" never goes with tags; the listing refuses both). +_TAG_MODES = ("all", "any", "untagged") _CONTROL = re.compile(r"[\x00-\x1f\x7f]") _SOURCE = re.compile(r"all|0|[1-9][0-9]{0,8}") # A collection id as the listing's ``collection`` query takes it (#331): none @@ -69,8 +70,9 @@ _STATE_FIELDS = { "category": ("text", 100, ""), "sort": ("choice", _LIBRARY_SORTS, "newest"), "tags": ("tags", None, ()), - # #444: "untagged" never goes with tags (the listing refuses both). + # #444: the Tags match and the output folder (relative, "/"-joined). "tag_mode": ("choice", _TAG_MODES, "all"), + "folder": ("text", 1024, ""), }, } diff --git a/py/api/library.py b/py/api/library.py index 5bd4e973..be699e70 100644 --- a/py/api/library.py +++ b/py/api/library.py @@ -21,9 +21,11 @@ from .pairs import pair_response try: from ...database.library import ARCHIVE_SCOPES from ...database.library_commands import LibraryCommandError + from ...database.library_folders import folder_prefixes, normalize_folder except ImportError: from database.library import ARCHIVE_SCOPES from database.library_commands import LibraryCommandError + from database.library_folders import folder_prefixes, normalize_folder _SECRET = os.urandom(32) @@ -283,13 +285,47 @@ class LibraryRoutesMixin: outputs = list(self._get_all_output_dirs()) return self._media_roots(outputs), len(outputs) + def _library_root_spellings(self) -> list[str]: + """Every spelling of each output root (#444 folder filter): the + validated, resolved root and each configured or detected path that + resolves to it, so images saved through a symlinked root match.""" + from ..config import GalleryConfig + + roots = [str(root) for root in self._get_all_output_dirs()] + candidates = [ + *GalleryConfig.MONITORING_DIRECTORIES, + self._find_comfyui_output_dir(), + *self._comfyui_output_dir_list(), + ] + spellings = [] + for root in roots: + for value in (root, *candidates): + if not value or not isinstance(value, (str, os.PathLike)): + continue + value = str(value) + same = value == root or os.path.normcase( + os.path.realpath(value) + ) == os.path.normcase(root) + if same and value not in spellings: + spellings.append(value) + return spellings + @pair_response async def get_library_options(self, request: web.Request) -> web.Response: - result = await self._run_in_executor(self.db.get_library_options) + include = request.query.get("include", "") + if include not in ("", "folders"): + raise ValueError("include must be empty or 'folders'") + if include: + # Folders are read only when Refine opens (a walk of linked paths). + roots = await self._run_in_executor(self._library_root_spellings) + result = await self._run_in_executor(self.db.get_library_options, roots) + else: + result = await self._run_in_executor(self.db.get_library_options) return web.json_response({"success": True, **result}) @pair_response async def list_library(self, request: web.Request) -> web.Response: + folder = normalize_folder(request.query.get("folder", "")) options = { "text": request.query.get("text", ""), "category": request.query.get("category"), @@ -299,21 +335,34 @@ class LibraryRoutesMixin: } limit = _page_limit(request) cursor = request.query.get("cursor") + roots = ( + await self._run_in_executor(self._library_root_spellings) if folder else [] + ) + prefixes = folder_prefixes(roots, folder) if folder else None def read() -> tuple[dict, dict]: with self.db.library_snapshot(include_ceiling=True) as snapshot: conn, epoch, ceiling = snapshot + # The folder and the roots it was resolved under join the scope. scope = _scope( - {"kind": "parents", "projection": 2, "epoch": epoch, **options} + { + "kind": "parents", + "projection": 2, + "epoch": epoch, + **options, + **({"folder": folder, "roots": roots} if folder else {}), + } ) result = self.db._list_library( conn, epoch, ceiling, **options, + folder_prefixes=prefixes, limit=limit, after=_decode(cursor, scope), ) + result["folder"] = folder records = self.db.get_library_previews( [parent["id"] for parent in result["items"]] ) diff --git a/tests/browser/workspace-library-filters.spec.mjs b/tests/browser/workspace-library-filters.spec.mjs new file mode 100644 index 00000000..2dbb7101 --- /dev/null +++ b/tests/browser/workspace-library-filters.spec.mjs @@ -0,0 +1,143 @@ +// #444 (D150): the Library's output-folder filter and multi-tag OR in "Refine this view", against the real server, and +// the Refine dialog against the concept's Filters dialog (app.js:1345-1352). +import {writeFile} from 'node:fs/promises'; +import {startCoverage,saveCoverage} from '../../tools/vendor/browser-coverage.mjs'; +import {test,expect} from './fixtures.mjs'; +import {captureCanonicalState} from './history-canonical-comparison.mjs'; +import {inventoryOf,diffInventory} from './library-editor-canonical-comparison.mjs'; + +const route='/prompt_manager/workspace#/library'; +const concept='/__test__/canonical/workspace'; +const cards=page=>page.locator('article[data-prompt-id]'); +const card=(page,text)=>cards(page).filter({has:page.getByText(text,{exact:true})}); +const toolbar=page=>page.locator('#workspace-main search.toolbar'); +const filtersButton=page=>toolbar(page).getByRole('button',{name:/^Filters(?: · \d)?$/}); +const refine=page=>page.getByRole('dialog',{name:'Refine this view',exact:true}); +const folder=page=>refine(page).getByRole('combobox',{name:'Folder',exact:true}); +const tags=page=>refine(page).getByRole('textbox',{name:'Tags',exact:true}); +const match=page=>refine(page).getByRole('combobox',{name:'Tag match',exact:true}); +const apply=page=>refine(page).getByRole('button',{name:'Apply filters',exact:true}); +const caption=page=>page.locator('#workspace-main .section-caption > span').first(); + +test.beforeEach(async({page,request,browserName})=>{ + await startCoverage(page,browserName);await expect(await request.post('/__test__/reset',{data:{native_library:true}})).toBeOK(); + page.libraryErrors=[];page.on('pageerror',error=>page.libraryErrors.push(error.message)); +}); +test.afterEach(async({page,browserName})=>{await saveCoverage(page,browserName);expect(page.libraryErrors).toEqual([]);}); + +async function open(page) { + await page.goto(route);await expect(caption(page)).toHaveText(/^3 positive prompts · /); +} +/** Open Refine and wait for the folder list it reads on opening. */ +async function openRefine(page) { + const folders=page.waitForResponse(item=>new URL(item.url()).pathname==='/prompt_manager/library/options'&&new URL(item.url()).searchParams.get('include')==='folders'); + await filtersButton(page).click();await expect(refine(page)).toBeVisible();await folders; +} + +test('Refine offers Folder, Tags and Tag match; a folder narrows the shelf to prompts with images there',async({page})=>{ + await open(page);await openRefine(page); + await expect(refine(page).locator('.field label')).toHaveText(['Folder','Tags','Tag match']); + // Both roots hold a "shared" folder; the root's own images name no folder. + await expect(folder(page).locator('option')).toHaveText(['All output folders','shared']); + await expect(match(page).locator('option')).toHaveText(['All of these tags','Any of these tags','Prompts without tags']); + await folder(page).selectOption('shared');await apply(page).click(); + await expect(caption(page)).toHaveText(/^1 positive prompt · /); + await expect(cards(page)).toHaveCount(1);await expect(card(page,'Cat portrait')).toHaveCount(1); + await expect(filtersButton(page)).toHaveText('Filters · 1'); + await openRefine(page);await expect(folder(page)).toHaveValue('shared'); + await refine(page).getByRole('button',{name:'Clear filters',exact:true}).click(); + await expect(caption(page)).toHaveText(/^3 positive prompts · /);await expect(filtersButton(page)).toHaveText('Filters'); +}); + +test('Tag match Any shows every prompt carrying one of the tags; changing only the match applies it',async({page})=>{ + await open(page);await openRefine(page); + await tags(page).fill('cats, dogs');await apply(page).click(); + await expect(caption(page)).toHaveText(/^0 positive prompts · /);await expect(filtersButton(page)).toHaveText('Filters · 1'); + await openRefine(page);await match(page).selectOption('any');await apply(page).click(); + await expect(caption(page)).toHaveText(/^2 positive prompts · /); + await expect(cards(page)).toHaveCount(2);await expect(card(page,'Cat portrait')).toHaveCount(1);await expect(card(page,'Dog landscape')).toHaveCount(1); + await expect(filtersButton(page)).toHaveText('Filters · 2'); +}); + +test('a saved view keeps its folder and tag match',async({page})=>{ + await open(page);await openRefine(page); + await folder(page).selectOption('shared');await tags(page).fill('cats, dogs');await match(page).selectOption('any');await apply(page).click(); + await expect(caption(page)).toHaveText(/^1 positive prompt · /); + await toolbar(page).getByRole('button',{name:'Save view',exact:true}).click(); + const save=page.getByRole('dialog',{name:'Save this view',exact:true}); + await save.getByRole('textbox',{name:'View name',exact:true}).fill('Shared cats or dogs'); + const created=page.waitForResponse(item=>new URL(item.url()).pathname==='/prompt_manager/saved-views'&&item.request().method()==='POST'); + await save.getByRole('button',{name:'Save',exact:true}).click(); + const body=(await created).request().postDataJSON(); + expect(body.state).toMatchObject({folder:'shared',tag_mode:'any',tags:['cats','dogs']}); + await page.goto('about:blank');await open(page); + await page.getByRole('navigation',{name:'Saved views',exact:true}).getByRole('link',{name:'Shared cats or dogs',exact:true}).click(); + await expect(caption(page)).toHaveText(/^1 positive prompt · /);await expect(filtersButton(page)).toHaveText('Filters · 3'); +}); + +test('a chosen folder missing from the folder list stays listed and chosen in Refine',async({page})=>{ + await open(page);await openRefine(page);await folder(page).selectOption('shared');await apply(page).click(); + await expect(caption(page)).toHaveText(/^1 positive prompt · /); + // The folder has no image any more as far as the list knows (or the read failed): Apply must not drop it. + await page.route('**/prompt_manager/library/options?include=folders',route=>route.fulfill({json:{success:true,library_epoch:'x',categories:[],folders:[]}})); + await openRefine(page); + await expect(folder(page).locator('option')).toHaveText(['All output folders','shared']);await expect(folder(page)).toHaveValue('shared'); + await apply(page).click();await expect(caption(page)).toHaveText(/^1 positive prompt · /);await expect(filtersButton(page)).toHaveText('Filters · 1'); +}); + +// Live B1: the server's flat list sorts full paths as text ("-" and "_" before "/"); Refine shows each folder directly +// under its own parent and keeps case variants apart. +test('the Folder list nests each folder under its exact parent',async({page})=>{ + const folders=['CCC','CCC/mabes9sans-dataset','CCC/mabes9sans-dataset-v1','CCC/mabes9sans-dataset/mabes9sans-dataset_IMG_GEN', + 'FLUX','FLUX/2025-12-11','flux','flux/images','flux-klein','kiko','kiko-v6','kiko/char'].sort(); + await page.route('**/prompt_manager/library/options?include=folders',route=>route.fulfill({json:{success:true,library_epoch:'x',categories:[],folders}})); + await open(page);await openRefine(page); + const I='\u00a0\u00a0'; + await expect(folder(page).locator('option')).toHaveText(['All output folders','CCC',`${I}mabes9sans-dataset`,`${I}${I}mabes9sans-dataset_IMG_GEN`, + `${I}mabes9sans-dataset-v1`,'FLUX',`${I}2025-12-11`,'flux',`${I}images`,'flux-klein','kiko',`${I}char`,'kiko-v6']); + expect(await folder(page).locator('option').evaluateAll(nodes=>nodes.map(node=>node.value))).toEqual(['','CCC','CCC/mabes9sans-dataset', + 'CCC/mabes9sans-dataset/mabes9sans-dataset_IMG_GEN','CCC/mabes9sans-dataset-v1','FLUX','FLUX/2025-12-11','flux','flux/images','flux-klein', + 'kiko','kiko/char','kiko-v6']); +}); + +test('an older server that ignores the folder shows an error with Retry, never the whole shelf',async({page})=>{ + await open(page); + await page.route('**/prompt_manager/library?*',async route=>{ + const response=await route.fetch();const body=await response.json();delete body.folder;delete body.tag_mode; + await route.fulfill({response,json:body}); + }); + await openRefine(page);await folder(page).selectOption('shared');await apply(page).click(); + await expect(page.locator('#workspace-main').getByText('Prompt library could not be loaded. Retry.')).toBeVisible(); + await expect(cards(page)).toHaveCount(0); +}); + +// D150: the Refine dialog is the concept's Filters dialog: its Folder select and Tags input, plus Tag match drawn as the +// concept's own select; the concept's "Prompt evidence" (no data yet) and its prototype note are not shown. +function mapFiltersDialog(reference,{folders,matchOptions}) { + const [conceptFolder,,conceptTags]=reference.fields; + return {...reference, + fields:[{...conceptFolder,options:folders},{...conceptTags},{...conceptFolder,label:'Tag match',options:matchOptions}], + small:[], + // Production's Apply reads "Apply filters" (Images and Library share it, #249). + actions:{...reference.actions,buttons:reference.actions.buttons.map(item=>item.text==='Apply tag filter'?{...item,text:'Apply filters'}:item)}}; +} +for(const [width,height] of [[1600,1100],[390,844]])for(const theme of ['dark','light'])test(`Refine dialog matches the concept Filters dialog ${width} ${theme} (D150)`,async({page},testInfo)=>{ + await page.setViewportSize({width,height}); + await page.goto('about:blank');await page.goto(`${concept}#library`);await page.getByRole('combobox',{name:'Theme',exact:true}).selectOption(theme); + await page.locator('#main').getByRole('button',{name:'Filters',exact:true}).click();await expect(page.locator('#modal')).toBeVisible(); + const reference=await page.locator('#modal').evaluate(inventoryOf); + await captureCanonicalState(page,testInfo,`concept-filters-${width}-${theme}`,'#modal'); + await open(page);await page.getByRole('combobox',{name:'Theme',exact:true}).selectOption(theme);await openRefine(page); + await expect(folder(page).locator('option')).toHaveCount(2); + const actual=await refine(page).evaluate(inventoryOf); + await captureCanonicalState(page,testInfo,`production-refine-${width}-${theme}`,'dialog[open]'); + const expected=mapFiltersDialog(reference,{folders:[['','All output folders'],['shared','shared']],matchOptions:[['all','All of these tags'],['any','Any of these tags'],['untagged','Prompts without tags']]}); + const path=testInfo.outputPath('refine-inventory.json');await writeFile(path,JSON.stringify({reference,actual},null,2)); + await testInfo.attach('refine-inventory',{path,contentType:'application/json'}); + expect(reference.fields.map(item=>item.label)).toEqual(['Folder','Prompt evidence','Tags']); + expect(diffInventory(expected,actual)).toEqual([]); + await refine(page).getByRole('button',{name:'Close dialog',exact:true}).click(); + await openRefine(page);await folder(page).selectOption('shared');await tags(page).fill('cats, dogs');await match(page).selectOption('any');await apply(page).click(); + await expect(caption(page)).toHaveText(/^1 positive prompt · /);await page.mouse.move(0,0); + await captureCanonicalState(page,testInfo,`production-library-folder-any-${width}-${theme}`,'#workspace-main'); +}); diff --git a/tests/browser/workspace-library.spec.mjs b/tests/browser/workspace-library.spec.mjs index 685eab8d..a2ba6187 100644 --- a/tests/browser/workspace-library.spec.mjs +++ b/tests/browser/workspace-library.spec.mjs @@ -155,7 +155,8 @@ test('Retry re-reads a failed category list with the shelf',async({page})=>{ await page.locator('#workspace-main search.toolbar').getByRole('button',{name:'Filters',exact:true}).click(); await refine(page).getByRole('button',{name:'Apply filters',exact:true}).click();await expect(refine(page)).toHaveCount(0); const search=page.getByRole('searchbox',{name:'Search workspace',exact:true});await search.fill(' '); - await page.waitForTimeout(400);expect(reads).toEqual([]);await expect(cards(page)).toHaveCount(3); + // Only the folder list Refine reads on opening (D150), never the shelf. + await page.waitForTimeout(400);expect(reads).toEqual(['/prompt_manager/library/options']);await expect(cards(page)).toHaveCount(3); }); test('failed library reads retry while late search responses cannot overwrite newer results',async({page})=>{ @@ -200,7 +201,7 @@ test('library toolbar matches the concept and tags live in Refine this view',asy // D44: the active Tags filter shows on the button; the category is the toolbar's own and is not counted. await expect(toolbar.getByRole('button')).toHaveText(['Save view','Filters · 1']); await toolbar.getByRole('button',{name:'Filters · 1',exact:true}).click(); - await expect(refine(page).locator('.field label')).toHaveText(['Tags','Tag match']);await expect(refine(page).getByRole('textbox',{name:'Tags',exact:true})).toHaveValue('studio'); + await expect(refine(page).locator('.field label')).toHaveText(['Folder','Tags','Tag match']);await expect(refine(page).getByRole('textbox',{name:'Tags',exact:true})).toHaveValue('studio'); await refine(page).getByRole('button',{name:'Clear filters',exact:true}).click();await expect(refine(page)).toHaveCount(0); await expect(toolbar.getByRole('button',{name:'Filters',exact:true})).toBeFocused(); await expect(page.getByRole('combobox',{name:'Filter by category',exact:true})).toHaveValue('');await expect(cards(page)).toHaveCount(3); diff --git a/tests/browser/workspace-saved-views.spec.mjs b/tests/browser/workspace-saved-views.spec.mjs index 26a2c1be..1b216a4d 100644 --- a/tests/browser/workspace-saved-views.spec.mjs +++ b/tests/browser/workspace-saved-views.spec.mjs @@ -208,7 +208,7 @@ test('a view stored under older rules still lists and opens with its usable valu await raw({ page: 'library', name: '', state: JSON.stringify({ q: 'dog', sort: 'name-desc' }) }); expect(await listed(request)).toEqual([ { id: imagesId, page: 'images', name: 'Old cats', state: { ...IMAGE_DEFAULTS, q: 'cat' } }, - expect.objectContaining({ page: 'library', name: 'Saved view', state: { q: 'dog', category: '', sort: 'newest', tags: [], tag_mode: 'all' } }), + expect.objectContaining({ page: 'library', name: 'Saved view', state: { q: 'dog', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } }), ]); // A state that is no JSON object is never listed, and its link behaves like a missing view. const corruptId = await raw({ page: 'images', name: 'Corrupt', state: '{not json' }); @@ -233,7 +233,7 @@ test('a view stored under older rules still lists and opens with its usable valu // D153 (#447): the workspace no longer offers "Highest rated"; the API still stores such a view as it was saved. test('a view saved under Highest rated opens on Recently added on Images and the Library', async ({ page, request }) => { const images = await create(request, { page: 'images', name: 'Rated cats', state: { ...IMAGE_DEFAULTS, q: 'cat', sort: 'rating' } }); - const library = await create(request, { page: 'library', name: 'Rated dogs', state: { q: 'dog', category: '', sort: 'rating', tags: [], tag_mode: 'all' } }); + const library = await create(request, { page: 'library', name: 'Rated dogs', state: { q: 'dog', category: '', sort: 'rating', tags: [], tag_mode: 'all', folder: '' } }); expect((await listed(request)).map(view => view.state.sort)).toEqual(['rating', 'rating']); const listings = []; page.on('request', item => { @@ -257,7 +257,7 @@ test('a view saved under Highest rated opens on Recently added on Images and the test('leaving a saved view returns Images and the Library to their own last state (#249 M2)', async ({ page, request }) => { const dogs = await create(request, { page: 'images', name: 'Dogs', state: { ...IMAGE_DEFAULTS, q: 'dogs' } }); - const cats = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all' } }); + const cats = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } }); await page.goto(`${workspace}#/images`); await search(page).fill('cat'); await expect(imageCards(page)).toHaveCount(1); @@ -300,7 +300,7 @@ test('a toolbar change while sources load keeps the saved Source and Folder (#24 }); test('a saved view left while its list or rename is pending leaves the next page alone', async ({ page, request }) => { - const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all' } }); + const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } }); let releaseList = () => {}; const holdList = async () => { const held = new Promise(resolve => { releaseList = resolve; }); @@ -348,7 +348,7 @@ test('an unknown saved view opens Images, and a failed listing offers Retry', as await expect(page).toHaveURL(/#\/images$/); await expect(main(page).getByRole('heading', { level: 1 })).toHaveText('Images'); - const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all' } }); + const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } }); let failing = true; await page.route(`**${API}`, route => failing && route.request().method() === 'GET' ? route.fulfill({ status: 500, contentType: 'application/json', body: '{"success":false,"error":"down"}' }) @@ -539,7 +539,7 @@ test('Edit renames an open view, and deletes it only after confirming inside the }); test('a deleted Library view returns to the Library, and failed edits say so', async ({ page, request }) => { - const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all' } }); + const view = await create(request, { page: 'library', name: 'Cats', state: { q: 'cat', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' } }); await page.goto(`${workspace}#/saved/${view.id}`); await expect(heading(page)).toHaveText('Cats'); const edit = main(page).locator('.heading-actions > button').first(); diff --git a/tests/js/workspace-library-state.test.js b/tests/js/workspace-library-state.test.js index f765a729..a31063d0 100644 --- a/tests/js/workspace-library-state.test.js +++ b/tests/js/workspace-library-state.test.js @@ -104,6 +104,20 @@ test('archived variant reads use their own scope and cursor without changing the const active=[];const plain=createLibraryDetail({api:{get:async url=>{active.push(url);return {positive:item(1),variants:[]};}},onChange:()=>{}});await plain.load(1);assert.equal(new URL(active[0],'https://x').searchParams.has('archive'),false); assert.throws(()=>createLibraryDetail({api:{},onChange:()=>{},archive:'parents'})); }); +test('tag mode any and a folder are sent, echoed and validated (#444)',async()=>{ + const {createLibraryStream,LIBRARY_TAG_MODES}=await source;assert.deepEqual(LIBRARY_TAG_MODES,['all','any','untagged']); + const calls=[];const pending=[{...page([item(1)]),tag_mode:'any',folder:'2026/Aug'},{...page([item(2)]),tag_mode:'all',folder:''},page([item(3)]),{...page([item(4)]),tag_mode:'all'}]; + const s=createLibraryStream({api:{get:async(url)=>{calls.push(new URL(url,'https://x').searchParams);return pending.shift();}}}); + assert.equal(await s.reset({tags:['a','b'],tagMode:'any',folder:'2026/Aug'}),1); + assert.equal(calls[0].get('tag_mode'),'any');assert.equal(calls[0].get('folder'),'2026/Aug');assert.deepEqual(calls[0].getAll('tag'),['a','b']); + // An older server that ignored them answers without the echo: an error, never the unfiltered shelf. + assert.equal(await s.reset({tags:['a','b'],tagMode:'any',folder:'2026/Aug'}),0);assert.ok(s.state.error instanceof TypeError); + // The defaults send neither, so they need no echo. + assert.equal(await s.reset({tagMode:'all',folder:''}),1);assert.equal(calls[2].has('tag_mode'),false);assert.equal(calls[2].has('folder'),false); + assert.equal(await s.reset({folder:'x'}),0);assert.ok(s.state.error instanceof TypeError,'a folder without its echo is refused'); + await assert.rejects(s.reset({tagMode:'or'}),TypeError);await assert.rejects(s.reset({folder:3}),TypeError); + assert.equal(calls.length,4);s.dispose(); +}); test('tag mode untagged is sent, echoed and never combined with tags (#444)',async()=>{ const {createLibraryStream}=await source;const calls=[];const pending=[{...page([item(1)]),tag_mode:'untagged'},page([item(2)]),page([item(3)])]; const s=createLibraryStream({api:{get:async(url)=>{calls.push(url);return pending.shift();}}}); @@ -112,7 +126,7 @@ test('tag mode untagged is sent, echoed and never combined with tags (#444)',asy assert.equal(await s.reset({tagMode:'untagged'}),0);assert.ok(s.state.error instanceof TypeError);assert.deepEqual(s.state.items,[]); // The default sends no tag_mode, so it needs no echo. assert.equal(await s.reset({tagMode:'all'}),1);assert.equal(new URL(calls[2],'https://x').searchParams.has('tag_mode'),false); - await assert.rejects(s.reset({tagMode:'untagged',tags:['a']}),TypeError);await assert.rejects(s.reset({tagMode:'none'}),TypeError); + await assert.rejects(s.reset({tagMode:'untagged',tags:['a']}),TypeError);await assert.rejects(s.reset({tagMode:'or'}),TypeError); assert.equal(calls.length,3);s.dispose(); }); test('the stream keeps the first page\'s storage epoch and refuses a later page from another one (#437)',async()=>{ @@ -131,3 +145,28 @@ test('the shelf no longer sorts by rating (D153)',async()=>{ assert.throws(()=>createLibraryStream({api:{get:async()=>page([])},restore:{filters:{sort:'rating'},state:{}}}),/Invalid library sort/); for(const sort of ['newest','oldest','name']) assert.equal(await s.reset({sort}),0);s.dispose(); }); +test('Refine › Folder nests each folder under its exact parent, by path segment (#444 live B1)',async()=>{ + const {libraryFolderOptions}=await source;assert.equal(typeof libraryFolderOptions,'function'); + const I='  '; + // The server's flat list sorts full paths as text, so "-" and "_" land before "/" (live report shapes). + const folders=['CCC','CCC/mabes9sans-dataset','CCC/mabes9sans-dataset-v1','CCC/mabes9sans-dataset/mabes9sans-dataset_IMG_GEN', + 'FLUX','FLUX/2025-12-11','flux','flux/images','flux-klein','illustrious','illustrious-base','illustrious/2026-10-26','illustrious/training', + 'illustrious/training/a','kiko','kiko-v6','kiko/char','kiko/XXX','QWEN','QWEN/x','Qwen','Qwen/y','Qwen-Training','Video','Video/v','video'].sort(); + const options=libraryFolderOptions(folders); + assert.deepEqual(options[0],['','All output folders']); + assert.deepEqual(options.slice(1),[ + ['CCC','CCC'],['CCC/mabes9sans-dataset',`${I}mabes9sans-dataset`],['CCC/mabes9sans-dataset/mabes9sans-dataset_IMG_GEN',`${I}${I}mabes9sans-dataset_IMG_GEN`], + ['CCC/mabes9sans-dataset-v1',`${I}mabes9sans-dataset-v1`], + ['FLUX','FLUX'],['FLUX/2025-12-11',`${I}2025-12-11`],['flux','flux'],['flux/images',`${I}images`],['flux-klein','flux-klein'], + ['illustrious','illustrious'],['illustrious/2026-10-26',`${I}2026-10-26`],['illustrious/training',`${I}training`],['illustrious/training/a',`${I}${I}a`], + ['illustrious-base','illustrious-base'], + ['kiko','kiko'],['kiko/char',`${I}char`],['kiko/XXX',`${I}XXX`],['kiko-v6','kiko-v6'], + ['QWEN','QWEN'],['QWEN/x',`${I}x`],['Qwen','Qwen'],['Qwen/y',`${I}y`],['Qwen-Training','Qwen-Training'], + ['Video','Video'],['Video/v',`${I}v`],['video','video'], + ]); + // Every parent precedes its children directly; a chosen folder missing from the list is kept, with its parents. + assert.deepEqual(libraryFolderOptions(['a'],'a/b/c').slice(1),[['a','a'],['a/b',`${I}b`],['a/b/c',`${I}${I}c`]]); + assert.deepEqual(libraryFolderOptions([],''),[['','All output folders']]); + // A POSIX folder whose name holds a backslash is one segment. + assert.deepEqual(libraryFolderOptions(['Metadata\\Klein_4b Wildcard Run']).slice(1),[['Metadata\\Klein_4b Wildcard Run','Metadata\\Klein_4b Wildcard Run']]); +}); diff --git a/tests/js/workspace-saved-views.test.js b/tests/js/workspace-saved-views.test.js index 72435079..bb9e0a50 100644 --- a/tests/js/workspace-saved-views.test.js +++ b/tests/js/workspace-saved-views.test.js @@ -5,7 +5,7 @@ const assert = require('node:assert/strict'); const source = import('../../web/js/workspace/saved-views.mjs'); const IMAGE_DEFAULTS = { q: '', category: '', sort: 'newest', source: 'all', folder: '', media: 'all', tags: [], favorites: false, collection: '' }; -const LIBRARY_DEFAULTS = { q: '', category: '', sort: 'newest', tags: [], tag_mode: 'all' }; +const LIBRARY_DEFAULTS = { q: '', category: '', sort: 'newest', tags: [], tag_mode: 'all', folder: '' }; test('a saved state always holds every page key and keeps only values its page accepts', async () => { const { normalizeViewState, SAVED_VIEW_PAGES } = await source; @@ -31,8 +31,8 @@ test('a saved state always holds every page key and keeps only values its page a assert.deepEqual(normalizeViewState('images', { ...full, q: 'x'.repeat(501), folder: 7, tags: Array(21).fill('t') }), { ...full, q: '', folder: '', tags: [] }); assert.deepEqual(normalizeViewState('images', { ...full, category: 'bad\u0007', tags: [' '] }), { ...full, category: '', tags: [] }); - // Library has no source, folder or media, and no Name Z–A. - assert.deepEqual(normalizeViewState('library', { ...full, sort: 'name-desc' }), { q: 'castle', category: 'Portraits', sort: 'newest', tags: ['moody', 'fog'], tag_mode: 'all' }); + // Library has no source or media, and no Name Z–A; its folder (#444) is the Library's own. + assert.deepEqual(normalizeViewState('library', { ...full, sort: 'name-desc' }), { q: 'castle', category: 'Portraits', sort: 'newest', tags: ['moody', 'fog'], tag_mode: 'all', folder: 'a/b' }); assert.deepEqual(normalizeViewState('library', { sort: 'oldest' }), { ...LIBRARY_DEFAULTS, sort: 'oldest' }); // The returned tags are a copy. const tags = ['one']; @@ -62,7 +62,7 @@ test('the listing keeps usable views in order and normalizes their state', async { id: 1, page: 'images', name: 'Videos', state: { media: 'video', source: '9' } }, ] }); assert.deepEqual(views, [ - { id: 3, page: 'library', name: 'Neon', state: { q: 'neon', category: '', sort: 'name', tags: [], tag_mode: 'all' } }, + { id: 3, page: 'library', name: 'Neon', state: { q: 'neon', category: '', sort: 'name', tags: [], tag_mode: 'all', folder: '' } }, { id: 1, page: 'images', name: 'Videos', state: { ...IMAGE_DEFAULTS, media: 'video', source: '9' } }, ]); assert.equal(parseSavedView(null), null); @@ -84,8 +84,8 @@ test('page filters and saved states convert both ways', async () => { assert.equal(imagesViewState({ ...filters, collection: undefined }).collection, ''); assert.equal(imagesViewState({ ...filters, collection: 7 }).collection, '7'); assert.equal(imagesViewState({ ...filters, favorites: undefined }).favorites, false); - const shelf = { text: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'], tagMode: 'all' }; - assert.deepEqual(libraryViewState(shelf), { q: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'], tag_mode: 'all' }); + const shelf = { text: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'], tagMode: 'any', folder: '2026' }; + assert.deepEqual(libraryViewState(shelf), { q: 'neon', category: 'Cities', sort: 'oldest', tags: ['night'], tag_mode: 'any', folder: '2026' }); assert.deepEqual(libraryFilters(libraryViewState(shelf)), shelf); }); @@ -216,16 +216,27 @@ test('a state over the API 4 KB cap is refused before anything is sent (#249 M5) assert.deepEqual(api.calls, []); }); -test('a Library view keeps its tag match, and untagged never travels with tags (#444)', async () => { +test('a Library view keeps its tag match and folder; an old view reads the defaults (#444)', async () => { + const { normalizeViewState, libraryFilters } = await source; + assert.deepEqual(normalizeViewState('library', { tag_mode: 'any', folder: '2026/Aug' }), { ...LIBRARY_DEFAULTS, tag_mode: 'any', folder: '2026/Aug' }); + assert.deepEqual(normalizeViewState('library', { tag_mode: 'or', folder: 'x'.repeat(1025) }), LIBRARY_DEFAULTS); + assert.deepEqual(libraryFilters({ q: 'a' }), { text: 'a', category: '', sort: 'newest', tags: [], tagMode: 'all', folder: '' }); +}); + +test('a Library view keeps untagged, and untagged never travels with tags (#444)', async () => { const { normalizeViewState, libraryFilters, libraryViewState } = await source; assert.equal(normalizeViewState('library', { tag_mode: 'untagged' }).tag_mode, 'untagged'); - assert.equal(normalizeViewState('library', { tag_mode: 'any' }).tag_mode, 'all'); assert.deepEqual(normalizeViewState('library', { tags: ['a'], tag_mode: 'untagged' }), { ...LIBRARY_DEFAULTS, tags: ['a'] }); - const shelf = { text: '', category: '', sort: 'newest', tags: [], tagMode: 'untagged' }; + const shelf = { text: '', category: '', sort: 'newest', tags: [], tagMode: 'untagged', folder: '' }; assert.equal(libraryViewState(shelf).tag_mode, 'untagged'); assert.deepEqual(libraryFilters(libraryViewState(shelf)), shelf); }); +test('a shelf without the #444 filters saves their defaults', async () => { + const { libraryViewState } = await source; + assert.deepEqual(libraryViewState({ text: '', category: '', sort: 'newest', tags: [] }), LIBRARY_DEFAULTS); +}); + test('a view saved under the retired Highest rated sort opens on Recently added (D153)', async () => { const { normalizeViewState, imagesFilters, libraryFilters } = await source; assert.equal(normalizeViewState('images', { sort: 'rating' }).sort, 'newest'); diff --git a/tests/test_api_library.py b/tests/test_api_library.py index 6239b611..e457a2cd 100644 --- a/tests/test_api_library.py +++ b/tests/test_api_library.py @@ -123,6 +123,72 @@ class TestLibraryAPI(ImageAPITestCase): }, ) + async def test_folder_filter_lists_and_matches_prompts_by_output_folder(self): + deep = self._save_prompt("deep") + top = self._save_prompt("top") + other = self._save_prompt("other") + self._save_prompt("no images") + for prompt, parts in ((deep, ("a", "b")), (top, ("a",)), (other, ("ab",))): + path = make_png(self.output_dir.joinpath(*parts, f"{prompt}.png")) + self.api.db.link_image_to_prompt(prompt, str(path)) + response, options = await self.get("/options", include="folders") + self.assertEqual(response.status, 200) + self.assertEqual(options["folders"], ["a", "a/b", "ab"]) + _, plain = await self.get("/options") + self.assertNotIn("folders", plain) + _, first = await self.get(folder="a", limit=1) + self.assertEqual((first["total"], first["folder"]), (2, "a")) + # Outer slashes are dropped (a backslash separates only on Windows). + outer = "\\a\\" if os.name == "nt" else "/a/" + _, second = await self.get(folder=outer, limit=1, cursor=first["next_cursor"]) + self.assertEqual( + sorted([first["items"][0]["text"], second["items"][0]["text"]]), + ["deep", "top"], + ) + _, all_rows = await self.get() + self.assertEqual((all_rows["total"], all_rows["folder"]), (4, "")) + for params in ( + {"folder": "a/../ab"}, + *([{"folder": "C:/x"}] if os.name == "nt" else []), + {"folder": "b", "cursor": first["next_cursor"]}, + {"cursor": first["next_cursor"]}, + ): + response, result = await self.get(limit=1, **params) + self.assertEqual(response.status, 400, params) + self.assertEqual(result["success"], False) + response, _ = await self.get("/options", include="tags") + self.assertEqual(response.status, 400) + + @unittest.skipIf(os.name == "nt", "a backslash is a separator on Windows") + async def test_a_posix_folder_name_with_a_backslash_filters_its_prompts(self): + prompt = self._save_prompt("klein") + name = "Metadata\\Klein_4b Wildcard Run" + path = make_png(self.output_dir / name / "x.png") + self.api.db.link_image_to_prompt(prompt, str(path)) + _, options = await self.get("/options", include="folders") + self.assertIn(name, options["folders"]) + _, result = await self.get(folder=name) + self.assertEqual((result["total"], result["folder"]), (1, name)) + + async def test_any_tag_mode_is_sent_through_and_echoed(self): + self._save_prompt("a only", tags=["a"]) + self._save_prompt("b only", tags=["b"]) + self._save_prompt("neither") + _, result = await self.get(tag=["a", "b"], tag_mode="any") + self.assertEqual(result["tag_mode"], "any") + self.assertEqual( + sorted(i["text"] for i in result["items"]), ["a only", "b only"] + ) + _, both = await self.get(tag=["a", "b"]) + self.assertEqual((both["total"], both["tag_mode"]), (0, "all")) + _, first = await self.get(tag=["a", "b"], tag_mode="any", limit=1) + response, _ = await self.get( + tag=["a", "b"], limit=1, cursor=first["next_cursor"] + ) + self.assertEqual(response.status, 400) + response, _ = await self.get(tag_mode="or") + self.assertEqual(response.status, 400) + async def test_variant_cursor_is_positive_scoped_and_required_limits_are_bounded( self, ): diff --git a/tests/test_library_folders.py b/tests/test_library_folders.py new file mode 100644 index 00000000..a49359fa --- /dev/null +++ b/tests/test_library_folders.py @@ -0,0 +1,243 @@ +"""#444: the Library's output-folder filter and multi-tag OR.""" + +import ntpath +import os +import posixpath +import tempfile +import unittest +from pathlib import Path + +from database.library_folders import ( + folder_prefixes, + normalize_folder, + relative_folder, +) +from database.operations import PromptDatabase +from utils.hashing import generate_prompt_hash + + +class TestFolderNames(unittest.TestCase): + def test_normalize_strips_outer_slashes_on_both_flavors(self): + for flavor in (posixpath, ntpath): + with self.subTest(flavor=flavor.__name__): + self.assertEqual(normalize_folder("", flavor), "") + self.assertEqual(normalize_folder("/", flavor), "") + self.assertEqual(normalize_folder("/a/b/", flavor), "a/b") + self.assertEqual(normalize_folder("..foo/b", flavor), "..foo/b") + + def test_a_backslash_separates_only_on_windows(self): + # Live B1: on POSIX a backslash is part of a folder name, as listed. + name = "Metadata\\Klein_4b Wildcard Run" + self.assertEqual(normalize_folder(name, posixpath), name) + self.assertEqual( + normalize_folder("2026\\08-Aug\\", posixpath), "2026\\08-Aug\\" + ) + self.assertEqual( + normalize_folder(name, ntpath), "Metadata/Klein_4b Wildcard Run" + ) + self.assertEqual(normalize_folder("2026\\08-Aug\\", ntpath), "2026/08-Aug") + # A colon names a drive only on Windows. + self.assertEqual(normalize_folder("a:b/c", posixpath), "a:b/c") + + def test_normalize_refuses_anything_but_a_folder_below_a_root(self): + for flavor, value in ( + *( + (flavor, value) + for flavor in (posixpath, ntpath) + for value in ("a/../b", "./a", "a//b", "a\x00b", 3, "x" * 1025) + ), + (ntpath, "C:/x"), + (ntpath, "c:"), + (ntpath, "a\\..\\b"), + ): + with self.subTest(flavor=flavor.__name__, value=value): + with self.assertRaises(ValueError): + normalize_folder(value, flavor) + + def test_the_default_flavor_is_the_platform_path_module(self): + self.assertEqual(normalize_folder("a\\b"), normalize_folder("a\\b", os.path)) + + def test_posix_prefixes_are_exact_and_end_in_a_separator(self): + self.assertEqual( + folder_prefixes(["/out", "/data/out/"], "a/b", posixpath), + ["/out/a/b/", "/data/out/a/b/"], + ) + self.assertEqual(folder_prefixes(["/"], "a", posixpath), ["/a/"]) + self.assertEqual( + folder_prefixes(["/out", "/out/"], "a", posixpath), ["/out/a/"] + ) + with self.assertRaises(ValueError): + folder_prefixes([], "a", posixpath) + + def test_windows_prefixes_fold_case_including_non_ascii(self): + self.assertEqual( + folder_prefixes(["C:\\ComfyUI\\Output"], "Ä/Sub", ntpath), + ["c:\\comfyui\\output\\ä\\sub\\"], + ) + self.assertEqual(folder_prefixes(["D:\\"], "x", ntpath), ["d:\\x\\"]) + self.assertEqual( + folder_prefixes(["\\\\Server\\Share\\Out"], "a", ntpath), + ["\\\\server\\share\\out\\a\\"], + ) + # The stored file_path is normcase(normpath(abspath)), the same fold. + stored = ntpath.normcase(ntpath.normpath("C:/ComfyUI/output/Ä/sub/x.png")) + self.assertTrue( + stored.startswith( + folder_prefixes(["C:/ComfyUI/Output"], "ä/SUB", ntpath)[0] + ) + ) + + def test_relative_folder_keeps_the_stored_spelling(self): + self.assertEqual( + relative_folder("/out/2026/Aug/x.png", ["/out"], posixpath), "2026/Aug" + ) + self.assertIsNone(relative_folder("/out/x.png", ["/out"], posixpath)) + self.assertIsNone(relative_folder("/outside/x.png", ["/out"], posixpath)) + self.assertIsNone(relative_folder("/output2/a/x.png", ["/out"], posixpath)) + self.assertEqual( + relative_folder("/out/..foo/x.png", ["/out"], posixpath), "..foo" + ) + self.assertEqual( + relative_folder("C:\\Out\\Ä\\Sub\\x.png", ["c:\\out"], ntpath), "Ä/Sub" + ) + self.assertEqual(relative_folder("D:\\a\\x.png", ["D:\\"], ntpath), "a") + + +class TestLibraryFolderAndAnyTag(unittest.TestCase): + def setUp(self): + self.temp = tempfile.TemporaryDirectory() + base = Path(os.path.realpath(self.temp.name)) + self.root = base / "output" + self.root.mkdir() + self.db = PromptDatabase(str(base / "folders.db")) + + def tearDown(self): + try: + self.db.close_all() + finally: + self.temp.cleanup() + + def save(self, text, **metadata): + return self.db.save_prompt( + text, prompt_hash=generate_prompt_hash(text), **metadata + ) + + def link(self, prompt_id, *parts): + path = self.root.joinpath(*parts) + path.parent.mkdir(parents=True, exist_ok=True) + path.write_bytes(b"x") + self.db.link_image_to_prompt(prompt_id, str(path)) + + def texts(self, **options): + return sorted(item["text"] for item in self.db.list_library(**options)["items"]) + + def in_folder(self, folder, roots=None): + return self.texts( + folder_prefixes=folder_prefixes(roots or [str(self.root)], folder) + ) + + def test_folder_matches_its_descendants_only(self): + self.link(self.save("deep"), "a", "b", "x.png") + self.link(self.save("top"), "a", "y.png") + self.link(self.save("sibling"), "ab", "z.png") + self.link(self.save("literal"), "100%_off", "w.png") + self.save("no images") + self.assertEqual(self.in_folder("a"), ["deep", "top"]) + self.assertEqual(self.in_folder("a/b"), ["deep"]) + self.assertEqual(self.in_folder("ab"), ["sibling"]) + self.assertEqual(self.in_folder("100%_off"), ["literal"]) + self.assertEqual(self.in_folder("1"), []) + cut = self.db.list_library( + folder_prefixes=folder_prefixes([str(self.root)], "a"), limit=1 + ) + self.assertEqual(cut["total"], 2) + + def test_folder_combines_with_text_category_and_tags(self): + self.link(self.save("red cat", category="Art", tags=["cat"]), "a", "1.png") + self.link(self.save("red dog", category="Art", tags=["dog"]), "a", "2.png") + self.link(self.save("red bird", category="Art", tags=["cat"]), "b", "3.png") + prefixes = folder_prefixes([str(self.root)], "a") + self.assertEqual( + self.texts(folder_prefixes=prefixes, tags=["cat"]), ["red cat"] + ) + self.assertEqual( + self.texts(folder_prefixes=prefixes, text="dog", category="Art"), + ["red dog"], + ) + + @unittest.skipIf(os.name == "nt", "symlinks need privileges on Windows") + def test_folder_listing_and_match_through_a_symlinked_root(self): + link = self.root.parent / "linked" + os.symlink(self.root, link) + prompt = self.save("via link") + (self.root / "a").mkdir() + (self.root / "a" / "x.png").write_bytes(b"x") + self.db.link_image_to_prompt(prompt, str(link / "a" / "x.png")) + spellings = [str(self.root), str(link)] + self.assertEqual(self.in_folder("a", spellings), ["via link"]) + self.assertEqual(self.in_folder("a", [str(self.root)]), []) + self.assertEqual(self.db.get_library_options(spellings)["folders"], ["a"]) + + @unittest.skipIf(os.name == "nt", "a backslash is a separator on Windows") + def test_a_posix_folder_with_a_backslash_lists_and_filters_alike(self): + name = "Metadata\\Klein_4b Wildcard Run" + self.link(self.save("klein"), name, "x.png") + self.link(self.save("other"), "Metadata", "y.png") + listed = self.db.get_library_options([str(self.root)])["folders"] + self.assertEqual(listed, ["Metadata", name]) + self.assertEqual(self.in_folder(normalize_folder(name)), ["klein"]) + self.assertEqual(self.in_folder(normalize_folder("Metadata")), ["other"]) + + def test_folder_listing_offers_ancestors_and_skips_outside_images(self): + self.link(self.save("deep"), "2026", "08-Aug", "x.png") + self.link(self.save("root image"), "y.png") + outside = Path(self.temp.name) / "elsewhere" / "z.png" + outside.parent.mkdir() + outside.write_bytes(b"x") + self.db.link_image_to_prompt(self.save("outside"), str(outside)) + options = self.db.get_library_options([str(self.root)]) + self.assertEqual(options["folders"], ["2026", "2026/08-Aug"]) + self.assertNotIn("folders", self.db.get_library_options()) + + def test_folder_combines_with_untagged(self): + self.link(self.save("plain in a"), "a", "1.png") + self.link(self.save("tagged in a", tags=["t"]), "a", "2.png") + self.link(self.save("plain in b"), "b", "3.png") + prefixes = folder_prefixes([str(self.root)], "a") + self.assertEqual( + self.texts(folder_prefixes=prefixes, tag_mode="untagged"), ["plain in a"] + ) + + def test_any_tag_mode_is_the_union_across_parent_and_pair(self): + self.save("parent a", tags=["a"]) + self.save("parent b", tags=["b"]) + self.save("both", tags=["a", "b"]) + self.save("neither", tags=["c"]) + self.db.save_prompt_pair("pair b", "noise", tags=["b"]) + split = self.save("split", tags=["a"]) + self.db.save_prompt_pair("split", "split noise", tags=["b"]) + archived = self.db.save_prompt_pair("archived b", "old", tags=["b"]) + self.db.archive_prompt_pair(archived) + self.assertIsInstance(split, int) + self.assertEqual( + self.texts(tags=["a", "b"], tag_mode="any"), + ["both", "pair b", "parent a", "parent b", "split"], + ) + self.assertEqual(self.texts(tags=["a", "b"], tag_mode="all"), ["both"]) + self.assertEqual(self.texts(tags=["a", "b"]), ["both"]) + for tags in (["a"], ["b"], ["c"]): + self.assertEqual( + self.texts(tags=tags, tag_mode="any"), self.texts(tags=tags) + ) + self.assertEqual( + self.texts(tags=["a", "b"], tag_mode="any", text="parent"), + ["parent a", "parent b"], + ) + self.assertEqual(self.texts(tag_mode="any"), self.texts()) + for mode in ("none", None, "ANY"): + with self.subTest(mode=mode), self.assertRaises(ValueError): + self.db.list_library(tag_mode=mode) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_saved_views.py b/tests/test_saved_views.py index d5cdc565..010c64cb 100644 --- a/tests/test_saved_views.py +++ b/tests/test_saved_views.py @@ -48,6 +48,7 @@ LIBRARY_STATE = { "sort": "rating", "tags": [], "tag_mode": "untagged", + "folder": "2026/Aug", } IMAGES_DEFAULTS = { "q": "", @@ -66,6 +67,7 @@ LIBRARY_DEFAULTS = { "sort": "newest", "tags": [], "tag_mode": "all", + "folder": "", } # A row saved by an older build: no favorites key, a retired key, a sort # that is no longer offered and an over-long folder. @@ -242,14 +244,15 @@ class TestSavedViewsApi(ImageAPITestCase): for sort in ("newest", "oldest", "name", "rating"): await self.create( page="library", - state={"sort": sort, "tags": ["a b"], "tag_mode": "all"}, + state={"sort": sort, "tags": ["a b"], "tag_mode": "all", "folder": ""}, ) - untagged = await self.create( - page="library", state={"tags": [], "tag_mode": "untagged"} - ) - self.assertEqual(untagged["state"]["tag_mode"], "untagged") + for mode in ("untagged", "any"): + view = await self.create( + page="library", state={"tags": [], "tag_mode": mode} + ) + self.assertEqual(view["state"]["tag_mode"], mode) _, listed = await self.send("GET") - self.assertEqual(len(listed["views"]), 21) + self.assertEqual(len(listed["views"]), 22) async def test_write_requires_the_collection_key(self): # #331 D140: a view saved on a collection page keeps the collection id. @@ -694,13 +697,24 @@ class TestSavedViewsSchema(unittest.TestCase): ) self.assertEqual(normalize_state("history", {}), {}) - def test_library_tag_mode_is_kept_and_never_paired_with_tags(self): - # A row from before #444 has no tag_mode: it opens matching every tag. + def test_library_tag_mode_and_folder_are_kept_with_defaults_for_old_rows(self): + # A row from before #444 has neither key. old = {"q": "", "category": "", "sort": "rating", "tags": ["a"]} self.assertEqual(normalize_state("library", old), {**LIBRARY_DEFAULTS, **old}) self.assertEqual( - normalize_state("library", {"tag_mode": "any"}), LIBRARY_DEFAULTS + normalize_state("library", {"tag_mode": "or", "folder": 3}), + LIBRARY_DEFAULTS, ) + for mode in ("all", "any", "untagged"): + self.assertEqual( + normalize_state("library", {"tag_mode": mode})["tag_mode"], mode + ) + with self.assertRaises(SavedViewInvalid): + encode_state("library", {**LIBRARY_STATE, "tag_mode": "or"}) + with self.assertRaises(SavedViewInvalid): + encode_state("library", {**LIBRARY_STATE, "folder": "x" * 1025}) + + def test_library_untagged_is_never_paired_with_tags(self): # Never written by this build: the tags win, as the listing refuses both. self.assertEqual( normalize_state("library", {"tags": ["a"], "tag_mode": "untagged"}), @@ -708,8 +722,6 @@ class TestSavedViewsSchema(unittest.TestCase): ) with self.assertRaises(SavedViewInvalid): encode_state("library", {**LIBRARY_STATE, "tags": ["a"]}) - with self.assertRaises(SavedViewInvalid): - encode_state("library", {**LIBRARY_STATE, "tag_mode": "none"}) def test_validation_errors_are_saved_view_invalid(self): for call in ( diff --git a/web/js/workspace/library-state.mjs b/web/js/workspace/library-state.mjs index bb4e4d98..596f82f7 100644 --- a/web/js/workspace/library-state.mjs +++ b/web/js/workspace/library-state.mjs @@ -12,18 +12,22 @@ function parameters(filters,limit) { for(const tag of filters.tags || []) if(tag.trim()) query.append('tag',tag.trim()); const sort=filters.sort || 'newest'; if(!['newest','oldest','name'].includes(sort)) throw new TypeError('Invalid library sort'); - // #444: how Tags match; `untagged` lists parents no tag's Browse returns and never goes with tags. - const tagMode=filters.tagMode ?? 'all'; + // #444: how Tags match (every tag, any of them, or parents no tag's Browse returns, which never goes with tags) and + // the output folder below a root. + const tagMode=filters.tagMode ?? 'all',folder=filters.folder ?? ''; if(!LIBRARY_TAG_MODES.includes(tagMode) || (tagMode==='untagged' && query.has('tag'))) throw new TypeError('Invalid library tag mode'); + if(typeof folder!=='string' || folder.length>1024) throw new TypeError('Invalid library folder'); if(tagMode!=='all') query.set('tag_mode',tagMode); + if(folder) query.set('folder',folder); query.set('sort',sort);return query; } /** The Library's tag match values (#444); the first is the default, sent as no parameter. */ -export const LIBRARY_TAG_MODES=Object.freeze(['all','untagged']); +export const LIBRARY_TAG_MODES=Object.freeze(['all','any','untagged']); // `variant_total` is the filtered parents' active negative variants (the concept caption's "M negative variants"). function validatePage(page,cursor,query) { - // A tag match the server did not echo was not applied (an older server): never show the whole shelf as filtered. + // A tag match or folder the server did not echo was not applied (an older server): never show the whole shelf as filtered. if(query?.has('tag_mode') && page?.tag_mode!==query.get('tag_mode')) throw new TypeError('Unconfirmed library tag mode'); + if(query?.has('folder') && page?.folder!==query.get('folder')) throw new TypeError('Unconfirmed library folder'); if(!page || !Array.isArray(page.items) || page.items.some(item=>!parent(item)) || !Number.isSafeInteger(page.total) || page.total<0 || !Number.isSafeInteger(page.variant_total) || page.variant_total<0 || typeof page.has_more!=='boolean' || !(page.next_cursor===null || (typeof page.next_cursor==='string' && page.next_cursor.length)) || page.has_more!==(page.next_cursor!==null) || (page.has_more && cursor===page.next_cursor)) throw new TypeError('Invalid library page'); } export function createLibraryStream({api,onChange=()=>{},restore}) { @@ -121,3 +125,28 @@ export function libraryWindow(count,columns,top,height,focused=-1) { if(focused>=0 && focused<count) keep.add(Math.floor(focused/columns)); return [...keep].sort((a,b)=>a-b).flatMap(row=>Array.from({length:columns},(_,column)=>row*columns+column).filter(index=>index<count)); } + +const byFolderName=(a,b)=>{const x=a.toLowerCase(),y=b.toLowerCase();return x<y?-1:x>y?1:a<b?-1:a>b?1:0;}; +/** + * Refine › Folder options (#444 D150): the concept's "All output folders" first, then the folders as a tree built by + * path segment, so each folder sits directly under its own parent (never under a longer sibling such as `kiko-v6`) and + * folders differing only in case stay distinct siblings. Siblings sort case-insensitively, then by exact spelling. A + * label is the last segment indented by depth (the legacy admin select). The chosen folder is kept even when the list + * lacks it, with its parents. + */ +export function libraryFolderOptions(folders,current='') { + const root=new Map(); + for(const folder of current?[...folders,current]:folders) { + let level=root; + for(const segment of folder.split('/')) {if(!level.has(segment)) level.set(segment,new Map());level=level.get(segment);} + } + const options=[['','All output folders']]; + (function walk(level,prefix,depth) { + for(const segment of [...level.keys()].sort(byFolderName)) { + const path=prefix?`${prefix}/${segment}`:segment; + options.push([path,`${'\u00a0\u00a0'.repeat(depth)}${segment}`]); + walk(level.get(segment),path,depth+1); + } + })(root,'',0); + return options; +} diff --git a/web/js/workspace/library.mjs b/web/js/workspace/library.mjs index e57169fc..82c12dc2 100644 --- a/web/js/workspace/library.mjs +++ b/web/js/workspace/library.mjs @@ -2,7 +2,7 @@ import {createScope,createButton} from '../shared/toolkit/core.js'; import {createPromptSender} from './prompt-sender.mjs'; import {mountPromptBlocks} from './prompt-inspector.mjs'; import {mountImages} from './images.mjs'; -import {createLibraryStream,createLibraryDetail,libraryPromptRoles,LIBRARY_TAG_MODES} from './library-state.mjs'; +import {createLibraryStream,createLibraryDetail,libraryPromptRoles,LIBRARY_TAG_MODES,libraryFolderOptions} from './library-state.mjs'; import {promptRoles} from './prompt-blocks.mjs'; import {createLibraryEditor,showWorkspaceToast} from './library-editor.mjs'; import {createLibraryEditorController} from './library-editor-state.mjs'; @@ -31,8 +31,8 @@ const metadataDrafts=new Map(); const archivedVisible=new Map(); // The concept's sorts first, then the Library's appended Oldest first (D14); D153 (#447) drops "Highest rated". const LIBRARY_SORTS=Object.freeze([['newest','Recently added'],['name','Name A–Z'],['oldest','Oldest first']]); -// Refine › Tag match (#444 D149), in LIBRARY_TAG_MODES order. -const TAG_MATCH_OPTIONS=Object.freeze([['all','All of these tags'],['untagged','Prompts without tags']]); +// Refine › Tag match (#444 D149, D150), in LIBRARY_TAG_MODES order. +const TAG_MATCH_OPTIONS=Object.freeze([['all','All of these tags'],['any','Any of these tags'],['untagged','Prompts without tags']]); // A card's focusable controls in order: the D148 selection checkbox, then its buttons (#437). const CARD_CONTROLS='button, input[type="checkbox"]'; const CATEGORIES_FAILED='Categories could not be loaded. Open the Prompt library again to retry.'; @@ -85,9 +85,9 @@ export function onLibraryWrite(listener) { * with the existing Library filters, and the heading takes focus as after a save. */ export function browseLibrary(filters) { - const {text='',category='',tags=[],sort='newest',tagMode='all'}=filters || {}; - if(typeof text!=='string' || typeof category!=='string' || !Array.isArray(tags) || tags.some(tag=>typeof tag!=='string') || typeof sort!=='string' || !LIBRARY_TAG_MODES.includes(tagMode)) throw new TypeError('Invalid Library filters'); - shelfVisit={filters:{text,category,tags:[...tags],sort,tagMode},anchor:null}; + const {text='',category='',tags=[],sort='newest',tagMode='all',folder=''}=filters || {}; + if(typeof text!=='string' || typeof category!=='string' || !Array.isArray(tags) || tags.some(tag=>typeof tag!=='string') || typeof sort!=='string' || !LIBRARY_TAG_MODES.includes(tagMode) || typeof folder!=='string') throw new TypeError('Invalid Library filters'); + shelfVisit={filters:{text,category,tags:[...tags],sort,tagMode,folder},anchor:null}; focusHeadingOnMount=true;window.location.hash='#/library'; } /** Remember the selection a reconciled save names, so the destination history opens on it once. */ @@ -160,16 +160,16 @@ export function mountLibrary(main,api,parentScope,{positiveId,savedState=null,on const visit=saved?null:shelfVisit, mountedGeneration=writeGeneration; // An invalidated visit keeps only filters and anchor: the stream restarts and heights are measured again. const restorable=visit?.state?visit:null; - const filters={text:'',category:'',tags:[],sort:'newest',tagMode:'all',...visit?.filters,...saved}; + const filters={text:'',category:'',tags:[],sort:'newest',tagMode:'all',folder:'',...visit?.filters,...saved}; // D153: a sort the shelf no longer offers (an older "Highest rated" visit) reads as Recently added. if(!LIBRARY_SORTS.some(([value])=>value===filters.sort)) filters.sort='newest'; const section=el('section',undefined,'library-shelf');section.setAttribute('aria-label','Prompt library'); // The shared concept toolbar (app.js toolbar(), a347–360): Tags live in "Refine this view" (D14), no Refresh (D15). // Its own scope, disposed after the visit is saved: removing it first would shorten the page and clamp scrollY. const toolbarScope=createScope(); - // D44: Filters reads "Filters · N" while Refine filters (from Refine or a Browse) narrow the shelf: Tags, and a - // Tag match other than All (#444 D149). - const refinements=()=>(filters.tags.length?1:0)+(filters.tagMode!=='all'?1:0); + // D44: Filters reads "Filters · N" while Refine filters (from Refine or a Browse) narrow the shelf: Tags, a Folder + // and a Tag match other than All (#444 D149, D150) each count once. + const refinements=()=>(filters.tags.length?1:0)+(filters.folder?1:0)+(filters.tagMode!=='all'?1:0); const toolbar=mountToolbar(section,{scope:toolbarScope,categories:filters.category?[filters.category]:[],sorts:LIBRARY_SORTS, value:{q:filters.text,category:filters.category,sort:filters.sort},activeFilters:refinements(),onChange:applyToolbar,onFilters:openFilters, onSaveView:trigger=>{const value=toolbar.value();onSaveView(trigger,libraryViewState({...filters,text:value.q.trim(),category:value.category,sort:value.sort}));}}); @@ -216,7 +216,7 @@ export function mountLibrary(main,api,parentScope,{positiveId,savedState=null,on // Nothing follows the grid in the concept; the legacy Library stays one click away on Overview (spec §5). continuation.append(message,retry,more);section.append(optionsStatus,exportStatus,selection.element,caption,grid,continuation);main.append(section); const cards=new Map(), rowNodes=new Map(), spacers=new Map(), heights=new Map(), pendingMeasurements=new Map(); - let state,frame=0,loadFrame=0,observer,optionsRequest,focusOverride=-1; + let state,frame=0,loadFrame=0,observer,optionsRequest,folderRequest,focusOverride=-1; const columns=()=>window.innerWidth>=1700?3:window.innerWidth<=700?1:2; const readInlineSize=()=>Math.round(grid.getBoundingClientRect().width*64)/64; let inlineSize=readInlineSize(), columnCount=columns(), layout={rows:[],height:0}, dirty=true, items, indexesById=new Map(); @@ -423,17 +423,26 @@ export function mountLibrary(main,api,parentScope,{positiveId,savedState=null,on if(Object.entries(next).every(([key,item])=>filters[key]===item)) return; Object.assign(filters,next);reset(); } - /** "Refine this view" with the Library's extra filters, Tags (D14) and Tag match (D149); Clear also clears the toolbar category. */ + /** + * "Refine this view" with the Library's filters: the concept's Folder (#444 D150, its options read when the dialog + * opens), Tags (D14) and Tag match (D149, D150); Clear also clears the toolbar category. + */ function openFilters(trigger) { - openRefineDialog({scope,trigger,fields:[{key:'tags'},{key:'tagMode',options:TAG_MATCH_OPTIONS}], - values:{tags:filters.tags.join(', '),tagMode:filters.tagMode},onApply:applyRefinement, - onClear:cleared=>{toolbar.setValue({category:cleared.category});applyRefinement(cleared);}}); + const refine=openRefineDialog({scope,trigger,fields:[{key:'folder',options:libraryFolderOptions([],filters.folder)},{key:'tags'}, + {key:'tagMode',options:TAG_MATCH_OPTIONS}],values:{folder:filters.folder,tags:filters.tags.join(', '),tagMode:filters.tagMode}, + onApply:applyRefinement,onClear:cleared=>{toolbar.setValue({category:cleared.category});applyRefinement(cleared);}}); + folderRequest?.abort();const request=folderRequest=new AbortController(); + // A failed read leaves "All output folders" and the current folder; nothing else depends on it. + api.get('/prompt_manager/library/options?include=folders',{signal:request.signal}).then(data=>{ + if(scope.signal.aborted || folderRequest!==request || !Array.isArray(data?.folders) || data.folders.some(folder=>typeof folder!=='string')) return; + refine.setOptions('folder',libraryFolderOptions(data.folders,filters.folder)); + },()=>{}); } - function applyRefinement({tags,tagMode='all',category=filters.category}) { + function applyRefinement({tags,tagMode='all',folder='',category=filters.category}) { // "Prompts without tags" takes no Tags (the dialog disables them). const next=tagMode==='untagged'?[]:[...new Set(tags.split(',').map(tag=>tag.trim()).filter(Boolean))]; - if(category===filters.category && tagMode===filters.tagMode && next.length===filters.tags.length && next.every((tag,index)=>tag===filters.tags[index])) return; - filters.category=category;filters.tags=next;filters.tagMode=tagMode;reset(); + if(category===filters.category && tagMode===filters.tagMode && folder===filters.folder && next.length===filters.tags.length && next.every((tag,index)=>tag===filters.tags[index])) return; + filters.category=category;filters.tags=next;filters.tagMode=tagMode;filters.folder=folder;reset(); } scope.listen(more,'click',()=>stream.loadMore()); // Retry also re-reads a category list that failed (D15: Refresh is gone). @@ -446,7 +455,7 @@ export function mountLibrary(main,api,parentScope,{positiveId,savedState=null,on // A write while mounted makes this stream stale: keep only what still means something. // A saved view is not the plain shelf, so it never replaces the Library return state (#249 M2, as Favorites). if(!saved) shelfVisit=mountedGeneration!==writeGeneration?{filters:savedFilters,anchor:savedAnchor}:state?{filters:savedFilters,state,scrollY:window.scrollY,layout:{inlineSize,columns:columnCount,heights:[...heights],anchor:savedAnchor}}:null; - toolbarScope.dispose();stream.dispose();optionsRequest?.abort();observer?.disconnect();measurementObserver?.disconnect();cancelAnimationFrame(frame);cancelAnimationFrame(loadFrame);for(const card of cards.values())removeCard(card);cards.clear();}); + toolbarScope.dispose();stream.dispose();optionsRequest?.abort();folderRequest?.abort();observer?.disconnect();measurementObserver?.disconnect();cancelAnimationFrame(frame);cancelAnimationFrame(loadFrame);for(const card of cards.values())removeCard(card);cards.clear();}); loadCategories();if(restorable) {render(stream.state);if(restorable.state.loading || restorable.state.error)stream.loadMore();} else reset(); } diff --git a/web/js/workspace/saved-views.mjs b/web/js/workspace/saved-views.mjs index 664bc4f0..4711c332 100644 --- a/web/js/workspace/saved-views.mjs +++ b/web/js/workspace/saved-views.mjs @@ -47,8 +47,9 @@ const FIELDS = Object.freeze({ }), library: Object.freeze({ q: text(500), category: text(100), sort: choice(['newest', 'oldest', 'name'], 'newest'), tags, - // #444: how the Tags match; 'untagged' never goes with tags (the listing refuses both). - tag_mode: choice(['all', 'untagged'], 'all'), + // #444: how the Tags match ('untagged' never goes with tags; the listing refuses both) and the output folder + // ('/'-joined, below a root). + tag_mode: choice(['all', 'any', 'untagged'], 'all'), folder: text(1024), }), }); @@ -117,11 +118,12 @@ export function imagesViewState(filters) { /** Library shelf filters (library.mjs `filters`) from a saved state, and the state of the current filters. */ export function libraryFilters(state) { const view = normalizeViewState('library', state); - return { text: view.q, category: view.category, sort: view.sort, tags: view.tags, tagMode: view.tag_mode }; + return { text: view.q, category: view.category, sort: view.sort, tags: view.tags, tagMode: view.tag_mode, folder: view.folder }; } export function libraryViewState(filters) { return normalizeViewState('library', { q: filters.text, category: filters.category, sort: filters.sort, tags: filters.tags, - tag_mode: filters.tagMode ?? 'all' }); + // A filter the shelf lacks (undefined) takes its key's default in normalizeViewState. + tag_mode: filters.tagMode, folder: filters.folder }); } /** diff --git a/web/js/workspace/workspace-dialogs.mjs b/web/js/workspace/workspace-dialogs.mjs index 8df9f351..9b114335 100644 --- a/web/js/workspace/workspace-dialogs.mjs +++ b/web/js/workspace/workspace-dialogs.mjs @@ -24,7 +24,7 @@ import { createButton, createSwitch } from '../shared/toolkit/core.js'; import { noun } from './plural.mjs'; -// `tagMode` (#444, D149): the Library's Tag match select after Tags; its `untagged` value disables Tags. +// `tagMode` (#444, D149, D150): the Library's Tag match select after Tags; its `untagged` value disables Tags. const FIELD_LABELS = Object.freeze({ source: 'Source', folder: 'Folder', media: 'Media', tags: 'Tags', tagMode: 'Tag match' }); const FIELD_ORDER = Object.keys(FIELD_LABELS);