From 1d4446082fbf4f3c4d1e76a41e92779555f12dc3 Mon Sep 17 00:00:00 2001 From: Vito Sansevero Date: Wed, 28 Oct 2026 20:27:53 -0700 Subject: [PATCH] fix(history): join outer-whitespace raw text to the history of the prompt it saves (#438) The Library stores text.strip() while a run keeps its exact raw text. The history window, run detail scope id, newest pair run and prompt hints now compare the raw text's stored form (one rule, utils/hashing.py, mirrored in SQLite with the same code points). Hints also keep matching the exact raw text, so older rows that kept outer whitespace stay hinted and protected. --- database/execution_contexts.py | 44 +++++++----- database/history_reads.py | 85 ++++++++++++++++-------- tests/comfyui/execution-history.spec.mjs | 5 ++ tests/test_execution_context_database.py | 53 ++++++++++++++- tests/test_execution_history_hooks.py | 15 +++++ tests/test_hashing.py | 43 ++++++++++++ tests/test_history_reads.py | 68 +++++++++++++++++-- utils/hashing.py | 37 +++++++++++ web/js/workspace/history.mjs | 2 +- 9 files changed, 301 insertions(+), 51 deletions(-) diff --git a/database/execution_contexts.py b/database/execution_contexts.py index 09483221..aeb4b801 100644 --- a/database/execution_contexts.py +++ b/database/execution_contexts.py @@ -7,6 +7,7 @@ from typing import Callable, ContextManager, Iterator try: from ..utils.execution_history_events import RunFinish, encode_command + from ..utils.hashing import stored_prompt_text, stored_prompt_text_sql from ..utils.execution_contexts import ( CAPTURE_REASONS, ContextBundle, @@ -15,6 +16,7 @@ try: ) except ImportError: from utils.execution_history_events import RunFinish, encode_command + from utils.hashing import stored_prompt_text, stored_prompt_text_sql from utils.execution_contexts import ( CAPTURE_REASONS, ContextBundle, @@ -134,7 +136,8 @@ def _insert_exact( def _role_hints(role: str) -> str: return ( - f"SELECT r.{role}_prompt_id, tx.text_value " + f"SELECT r.{role}_prompt_id, tx.text_value, " + f"{stored_prompt_text_sql('tx.text_value')} " "FROM execution_run_contexts r " "JOIN execution_contexts c ON c.context_id = r.context_id " f"JOIN execution_prompt_snapshots s ON s.snapshot_id = c.{role}_snapshot_id " @@ -146,21 +149,21 @@ def _role_hints(role: str) -> str: # Set form of the read-time prompt-hint rule in get_execution_contexts(): # a stored role hint is live only in the current storage epoch, and only when -# its exact raw snapshot text matches exactly one prompt, which is the hinted -# one (_prompt_hint). Evaluated once per statement, never per prompt row. +# exactly one prompt has its raw snapshot text or that text's stored form +# (outer whitespace stripped, #438), and that is the hinted one (_prompt_hint). +# Evaluated once per statement, never per prompt row. # Defines the CTE ``live_hint_prompts(prompt_id)``; prefix it with WITH. LIVE_HINT_CTES = ( - "stored_hints(prompt_id, raw) AS (" + "stored_hints(prompt_id, raw, stored) AS (" + _role_hints("positive") + " UNION " + _role_hints("negative") - + "), unique_hint_texts(prompt_id, raw) AS (" - "SELECT MIN(id), text COLLATE BINARY FROM prompts " - "WHERE text COLLATE BINARY IN (SELECT raw FROM stored_hints) " - "GROUP BY text COLLATE BINARY HAVING COUNT(*) = 1" + + "), hint_matches(prompt_id, raw, stored, match_id) AS (" + "SELECT h.prompt_id, h.raw, h.stored, p.id FROM stored_hints h " + "JOIN prompts p ON p.text COLLATE BINARY IN (h.raw, h.stored)" "), live_hint_prompts(prompt_id) AS (" - "SELECT h.prompt_id FROM stored_hints h JOIN unique_hint_texts u " - "ON u.prompt_id = h.prompt_id AND u.raw = h.raw COLLATE BINARY)" + "SELECT prompt_id FROM hint_matches GROUP BY prompt_id, raw, stored " + "HAVING COUNT(*) = 1 AND MIN(match_id) = prompt_id)" ) LIVE_HINT_PROMPT_IDS_SQL = ( "WITH " + LIVE_HINT_CTES + " SELECT DISTINCT prompt_id FROM live_hint_prompts" @@ -168,10 +171,16 @@ LIVE_HINT_PROMPT_IDS_SQL = ( def _prompt_hint(conn: sqlite3.Connection, raw: str | None) -> int | None: + """The one prompt whose text is ``raw`` or the stored form of it (#438). + + The Library stores ``text.strip()``; the exact form keeps hints of older + rows whose stored text kept outer whitespace. + """ if raw is None: return None rows = conn.execute( - "SELECT id FROM prompts WHERE text=? COLLATE BINARY LIMIT 2", (raw,) + "SELECT id FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 2", + (raw, stored_prompt_text(raw)), ).fetchall() return rows[0][0] if len(rows) == 1 else None @@ -180,23 +189,26 @@ def _hints( conn: sqlite3.Connection, positive: str | None, negative: str | None ) -> tuple: pos_id, neg_id = _prompt_hint(conn, positive), _prompt_hint(conn, negative) + stored = None if negative is None else stored_prompt_text(negative) pair_id = None negative_ambiguous = ( - negative is not None + stored is not None and neg_id is None and conn.execute( - "SELECT 1 FROM prompts WHERE text=? COLLATE BINARY LIMIT 1", (negative,) + "SELECT 1 FROM prompts WHERE text COLLATE BINARY IN (?,?) LIMIT 1", + (negative, stored), ).fetchone() is not None ) - if pos_id is not None and negative is not None and not negative_ambiguous: + if pos_id is not None and stored is not None and not negative_ambiguous: rows = conn.execute( "SELECT pp.id FROM prompt_pairs pp LEFT JOIN prompts n ON " "n.id=pp.negative_prompt_id " "WHERE pp.positive_prompt_id=? AND ((pp.negative_prompt_id IS NULL AND " "?='') " - "OR (pp.negative_prompt_id=? AND n.text=? COLLATE BINARY)) LIMIT 2", - (pos_id, negative, neg_id, negative), + "OR (pp.negative_prompt_id=? AND n.text COLLATE BINARY IN (?,?))) " + "LIMIT 2", + (pos_id, stored, neg_id, negative, stored), ).fetchall() if len(rows) == 1: pair_id = rows[0][0] diff --git a/database/history_reads.py b/database/history_reads.py index a96227fd..b803f55e 100644 --- a/database/history_reads.py +++ b/database/history_reads.py @@ -15,6 +15,11 @@ from .execution_history import HistoryEpochChanged from .execution_outputs import _capture, _REFERENCE_COLUMNS from .library import _ELIGIBLE +try: + from ..utils.hashing import stored_prompt_text, stored_prompt_text_sql +except ImportError: + from utils.hashing import stored_prompt_text, stored_prompt_text_sql + RunKey = tuple[int, str, int, str] @@ -100,14 +105,43 @@ _ORDER = ( ) # Stale-hint rows a newest-pair lookup inspects before it reports none. _PAIR_RUN_CANDIDATES = 16 +# The recorded texts of one scope, as a JSON array parameter (_scope_text_ids). +_IN_SCOPE = "IN (SELECT value FROM json_each(?))" _MATCHING = ( "SELECT DISTINCT rc.run_id FROM execution_prompt_snapshots s " "JOIN execution_contexts c ON c.positive_snapshot_id=s.snapshot_id " "JOIN execution_run_contexts rc ON rc.context_id=c.context_id " - "WHERE s.raw_text_id=?" + f"WHERE s.raw_text_id {_IN_SCOPE}" ) +def _scope_text_ids(conn: sqlite3.Connection, scope: str) -> str: + """Every recorded raw text whose stored form is ``scope``, as a JSON array. + + The Library stores ``text.strip()`` while a run keeps its exact raw text, + so a raw text joins the scope of the prompt it saves (#438). The values + are compared, never only their digests. + """ + return json.dumps( + [ + row[0] + for row in conn.execute( + # Only snapshot raw texts: effective and affix texts are skipped + "SELECT t.text_id FROM (SELECT DISTINCT raw_text_id AS text_id " + "FROM execution_prompt_snapshots) s CROSS JOIN " + "execution_context_texts t WHERE t.text_id=s.text_id AND " + f"{stored_prompt_text_sql('t.text_value')}=? COLLATE BINARY", + (scope,), + ) + ] + ) + + +def _scope_text_id(raw: str | None) -> str | None: + """The scope digest of a raw text: that of the text the Library stores.""" + return None if raw is None else content_digest("text", (stored_prompt_text(raw),)) + + def _limit(limit: int) -> None: if type(limit) is not int or not 1 <= limit <= 100: raise ValueError("History limit must be an integer from 1 to 100") @@ -255,7 +289,7 @@ class HistoryReadOperations: if saved is None: return None text = saved["text"] - raw_id = None if text is None else content_digest("text", (text,)) + raw_id = _scope_text_id(text) if after is not None and after.raw_text_id != raw_id: raise ValueError("History position text scope changed") group = { @@ -273,18 +307,11 @@ class HistoryReadOperations: next_position=None, resume_position=None, ) - if ( - raw_id is None - or conn.execute( - "SELECT 1 FROM execution_context_texts WHERE text_id=? " - "AND text_value=? COLLATE BINARY", - (raw_id, text), - ).fetchone() - is None - ): + if raw_id is None: return result + scope = _scope_text_ids(conn, stored_prompt_text(text)) where = f"r.run_id IN ({_MATCHING})" - params = [raw_id] + params = [scope] if after is not None: where += f" AND {_KEY}<=(?,?,?,?)" params.extend(after.upper) @@ -338,9 +365,9 @@ class HistoryReadOperations: "LEFT JOIN execution_prompt_snapshots n " "ON n.snapshot_id=c.negative_snapshot_id " "LEFT JOIN execution_context_texts t ON t.text_id=n.raw_text_id " - f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id=? " + f"WHERE rc.run_id IN ({placeholders}) AND p.raw_text_id {_IN_SCOPE} " "ORDER BY rc.anchor_id COLLATE BINARY,rc.run_id COLLATE BINARY", - (*by_run, raw_id), + (*by_run, scope), ) summaries = {} evidence_by_context = {} @@ -438,6 +465,7 @@ class HistoryReadOperations: if saved["text"] is None: return {"run": None} epoch = self.get_execution_epoch() + stored = stored_prompt_text(saved["text"]) candidates = conn.execute( "SELECT r.run_id,rc.anchor_id,rc.context_id," "c.positive_snapshot_id,c.negative_snapshot_id " @@ -445,10 +473,11 @@ class HistoryReadOperations: "ON rc.run_id=r.run_id JOIN execution_contexts c " "ON c.context_id=rc.context_id JOIN execution_prompt_snapshots s " "ON s.snapshot_id=c.positive_snapshot_id " - "WHERE s.raw_text_id=? AND rc.pair_id=? AND rc.reference_epoch=? " + f"WHERE s.raw_text_id {_IN_SCOPE} AND rc.pair_id=? " + "AND rc.reference_epoch=? " f"ORDER BY {_ORDER},rc.anchor_id COLLATE BINARY LIMIT ?", ( - content_digest("text", (saved["text"],)), + _scope_text_ids(conn, stored), pair_id, epoch, _PAIR_RUN_CANDIDATES, @@ -457,7 +486,11 @@ class HistoryReadOperations: for row in candidates: positive = _values(conn, row["positive_snapshot_id"]) negative = _values(conn, row["negative_snapshot_id"]) - if positive is None or positive["raw"] != saved["text"]: + if ( + positive is None + or positive["raw"] is None + or stored_prompt_text(positive["raw"]) != stored + ): continue hints = _hints(conn, positive["raw"], negative["raw"] if negative else None) if hints[2] == pair_id: @@ -477,11 +510,9 @@ class HistoryReadOperations: snapshots = { row["anchor_id"]: row for row in conn.execute( - "SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id," - "p.raw_text_id AS positive_raw_text_id " + "SELECT rc.anchor_id,c.positive_snapshot_id,c.negative_snapshot_id " "FROM execution_run_contexts rc JOIN execution_contexts c " - "ON c.context_id=rc.context_id LEFT JOIN execution_prompt_snapshots p " - "ON p.snapshot_id=c.positive_snapshot_id WHERE rc.run_id=?", + "ON c.context_id=rc.context_id WHERE rc.run_id=?", (run_id,), ) } @@ -491,12 +522,12 @@ class HistoryReadOperations: context.update( { key: row[key] - for key in ( - "positive_snapshot_id", - "negative_snapshot_id", - "positive_raw_text_id", - ) - } + for key in ("positive_snapshot_id", "negative_snapshot_id") + }, + # The history scope this context belongs to (#438) + positive_raw_text_id=_scope_text_id( + (context["positive"] or {}).get("raw") + ), ) pair = context["pair_id"] context["navigation"] = { diff --git a/tests/comfyui/execution-history.spec.mjs b/tests/comfyui/execution-history.spec.mjs index 09285e63..eab2b16e 100644 --- a/tests/comfyui/execution-history.spec.mjs +++ b/tests/comfyui/execution-history.spec.mjs @@ -309,6 +309,11 @@ test('fresh registered PM variants preserve exact raw whitespace and known empty expect(value.negative.raw).toBe(''); expect(value.negative.effective).toBe(''); expect(value.evidence.positive.raw).toMatchObject({ provenance: 'recorded', source: 'runtime_return' }); + // #438: the Library stores the stripped text; the run is in that prompt's history + expect(value.positive_prompt_id, 'the run names the prompt the Library saved').toEqual(expect.any(Number)); + const runId = (await readExecutionRuns(execution.databasePath, id)).rows[0].run_id; + const response = await execution.request.get(`/prompt_manager/library/${value.positive_prompt_id}/history`);expect(response.ok()).toBe(true); + expect((await response.json()).items.map(item => item.run_id)).toContain(runId); } }); diff --git a/tests/test_execution_context_database.py b/tests/test_execution_context_database.py index 64f87cd1..b9c7325f 100644 --- a/tests/test_execution_context_database.py +++ b/tests/test_execution_context_database.py @@ -474,6 +474,54 @@ class ContextDatabaseTests(unittest.TestCase): rotate_history_epoch(conn) self.assertEqual(assert_same(), set()) + def test_hints_resolve_the_stored_form_of_outer_whitespace_raw_text(self): + # #438: the Library stores text.strip(); the raw run text keeps its spaces. + db = self.database() + conn = db.model.get_connection() + with conn: + conn.execute( + "INSERT INTO prompts(id,text) VALUES(1,'positive'),(2,'negative')" + ) + conn.execute( + "INSERT INTO prompt_pairs(id,positive_prompt_id,negative_prompt_id) " + "VALUES(7,1,2)" + ) + start = self.capture(db, 0, bundle(" negative\n", positive="positive\n\t")) + context = db.get_execution_contexts(start.run_id)["contexts"][0] + self.assertEqual(context["positive"]["raw"], "positive\n\t") + self.assertEqual( + ( + context["positive_prompt_id"], + context["negative_prompt_id"], + context["pair_id"], + ), + (1, 2, 7), + ) + live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + self.assertEqual(live, {1, 2}) + stored = conn.execute( + "SELECT positive_prompt_id FROM execution_run_contexts WHERE run_id=?", + (start.run_id,), + ).fetchone()[0] + self.assertEqual(stored, 1) + # A legacy row that kept outer whitespace keeps its exact-text hint and + # with it the recorded-run protection from pruning. + with conn: + conn.execute("INSERT INTO prompts(id,text) VALUES(3,' legacy ')") + legacy = self.capture(db, 1, bundle("negative", positive=" legacy ")) + hinted = db.get_execution_contexts(legacy.run_id)["contexts"][0] + self.assertEqual(hinted["positive_prompt_id"], 3) + live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + self.assertEqual(live, {1, 2, 3}) + self.assertEqual(db.prune_orphaned_prompts(), 0) + # Both forms present make the hint ambiguous, in both rule forms. + with conn: + conn.execute("INSERT INTO prompts(id,text) VALUES(4,'legacy')") + hinted = db.get_execution_contexts(legacy.run_id)["contexts"][0] + self.assertIsNone(hinted["positive_prompt_id"]) + live = {row[0] for row in conn.execute(LIVE_HINT_PROMPT_IDS_SQL)} + self.assertEqual(live, {1, 2}) + def test_hints_require_exact_unique_raw_text_and_never_refresh(self): db = self.database() conn = db.model.get_connection() @@ -504,12 +552,13 @@ class ContextDatabaseTests(unittest.TestCase): db.get_execution_contexts(fresh.run_id)["contexts"][0]["positive_prompt_id"] ) - def test_null_negative_pair_only_matches_known_exact_empty(self): + def test_null_negative_pair_only_matches_a_known_stored_empty(self): db = self.database() with db.model.get_connection() as conn: conn.execute("INSERT INTO prompts(id,text) VALUES(1,'positive')") conn.execute("INSERT INTO prompt_pairs(id,positive_prompt_id) VALUES(7,1)") - for item, raw, expected in ((0, "", 7), (1, " ", None), (2, None, None)): + # A whitespace-only negative is the stored empty one (#438) + for item, raw, expected in ((0, "", 7), (1, " \n", 7), (2, None, None)): candidate = bundle(negative=raw or "") if raw is None: ctx = candidate.contexts[0] diff --git a/tests/test_execution_history_hooks.py b/tests/test_execution_history_hooks.py index 855c4e91..dbb03655 100644 --- a/tests/test_execution_history_hooks.py +++ b/tests/test_execution_history_hooks.py @@ -916,3 +916,18 @@ class CachedRuntimeHistoryTests(HooksCase): from database.run_attribution import named_prompt self.assertIsNone(named_prompt(self.db, self.rows()[1]["run_id"])) + + +class CachedRuntimeWhitespaceHistoryTests(CachedRuntimeHistoryTests): + """Raw text with outer whitespace joins the history of the prompt it saves (#438). + + The Library stores ``"a lighthouse the LLM wrote"``; the run keeps the exact + frozen raw text, newline included, yet is a tracked run of that prompt. + """ + + TEXT = "\u3000 a lighthouse the LLM wrote\n" + + def test_the_library_stores_the_stripped_text(self): + self.assertEqual( + self.db.get_prompt_by_id(self.prompt_id)["text"], self.TEXT.strip() + ) diff --git a/tests/test_hashing.py b/tests/test_hashing.py index ad02cdfc..a06eec86 100644 --- a/tests/test_hashing.py +++ b/tests/test_hashing.py @@ -16,6 +16,9 @@ from tests.background_stops import stop_tracker # noqa: E402 from utils.hashing import ( generate_prompt_hash, is_duplicate_prompt, + STRIPPED_CODE_POINTS, + stored_prompt_text, + stored_prompt_text_sql, ) ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) @@ -115,5 +118,45 @@ class TestGeneratePromptHashEdgeCases(unittest.TestCase): self.assertEqual(len(h), 64) +class TestStoredPromptText(unittest.TestCase): + """SQLite strips exactly the characters Python's str.strip() does (#438).""" + + WHITESPACE = "".join(c for c in map(chr, range(0x110000)) if c.isspace()) + + def sql_strip(self, value): + import sqlite3 + + conn = sqlite3.connect(":memory:") + try: + return conn.execute( + f"SELECT {stored_prompt_text_sql('?')}", (value,) + ).fetchone()[0] + finally: + conn.close() + + def test_code_points_are_exactly_python_whitespace(self): + self.assertEqual("".join(map(chr, STRIPPED_CODE_POINTS)), self.WHITESPACE) + + def test_python_rule_is_the_hash_strip(self): + self.assertEqual(stored_prompt_text(" a lighthouse\n"), "a lighthouse") + self.assertEqual( + generate_prompt_hash(stored_prompt_text(" A b\u3000")), + generate_prompt_hash(" A b\u3000"), + ) + + def test_sql_strips_every_python_whitespace_character(self): + for char in self.WHITESPACE: + with self.subTest(code=hex(ord(char))): + value = f"{char}a{char}b{char}" + self.assertEqual(self.sql_strip(value), stored_prompt_text(value)) + mixed = self.WHITESPACE + "a lighthouse é" + self.WHITESPACE + self.assertEqual(self.sql_strip(mixed), "a lighthouse é") + + def test_sql_keeps_what_python_keeps(self): + for value in ("\u200ba\u200b", "\ufeffa", "a\u0301", "", "x"): + with self.subTest(value=repr(value)): + self.assertEqual(self.sql_strip(value), stored_prompt_text(value)) + + if __name__ == "__main__": unittest.main() diff --git a/tests/test_history_reads.py b/tests/test_history_reads.py index e9e6a408..c02ea1e8 100644 --- a/tests/test_history_reads.py +++ b/tests/test_history_reads.py @@ -9,7 +9,12 @@ from unittest.mock import patch from database.execution_history import HistoryEpochChanged from database.execution_history_schema import history_schema_error -from utils.execution_contexts import ContextBundle, RecipeContext, TextEvidence +from utils.execution_contexts import ( + ContextBundle, + RecipeContext, + TextEvidence, + content_digest, +) from utils.execution_outputs import OutputBundle from tests.test_execution_context_database import bundle, snapshot from tests.test_execution_output_database import bundle as outputs @@ -146,10 +151,11 @@ class HistoryReadTests(unittest.TestCase): ) self.require_reads(db) with db.recorded_history_snapshot(): - for i in range(1, len(values) + 1): - self.assertEqual( - db.list_history_window(i)["recorded_run_count"], 2 if i == 1 else 1 - ) + # Outer whitespace joins the stored form's scope (#438); case and + # Unicode normalization stay distinct. + expected = (4, 1, 4, 4, 1, 1, 2, 2) + for i, count in enumerate(expected, 1): + self.assertEqual(db.list_history_window(i)["recorded_run_count"], count) self.assertIsNone(db.list_history_window(100)) self.assertIsNone(db.list_history_window(101)) page = db.list_history_window(99) @@ -989,3 +995,55 @@ class HistoryReadTests(unittest.TestCase): db.delete_prompt(saved["positive_prompt_id"]) with db.recorded_history_snapshot(): self.assertIsNone(db.get_newest_pair_run(saved["positive_prompt_id"], pair)) + + def test_outer_whitespace_raw_text_joins_the_stored_prompt_scope(self): + # #438: the Library stores text.strip(); the run keeps its raw text. + db = self.database() + positive = db.save_prompt("a lighthouse") + raws = ("a lighthouse\n", "a lighthouse", "　 a lighthouse \t") + starts = [ + self.capture(db, i, bundle(positive=raw), time=100 + i) + for i, raw in enumerate(raws) + ] + self.saved(db, "a lighthouse at dusk", 50) + self.capture(db, 9, bundle(positive="a lighthouse at dusk\n"), time=200) + with db.recorded_history_snapshot(): + page = db.list_history_window(positive) + first = db.list_history_window(positive, limit=2) + rest = db.list_history_window( + positive, limit=2, after=first["next_position"] + ) + details = [db.get_history_run(start.run_id) for start in starts] + self.assertEqual(page["recorded_run_count"], 3) + self.assertEqual(page["group"]["scope_status"], "matched") + self.assertEqual( + page["group"]["raw_text_id"], content_digest("text", ("a lighthouse",)) + ) + self.assertEqual( + [r["run_id"] for r in page["items"]], [s.run_id for s in starts] + ) + self.assertTrue(all(r["matching_anchors"] for r in page["items"])) + self.assertEqual(len(page["context_summaries"]), 3) + self.assertEqual( + [r["run_id"] for r in first["items"] + rest["items"]], + [s.run_id for s in starts[1:]] + [starts[0].run_id], + ) + for raw, detail in zip(raws, details): + context = detail["contexts"][0] + self.assertEqual(context["positive"]["raw"], raw) + self.assertEqual( + context["positive_raw_text_id"], page["group"]["raw_text_id"] + ) + self.assertEqual(context["positive_prompt_id"], positive) + self.assertTrue(context["navigation"]["positive_available"]) + + def test_newest_pair_run_follows_outer_whitespace_raw_text(self): + db = self.database() + pair = db.save_prompt_pair("a lighthouse", "fog") + positive = db.get_prompt_pair(pair)["positive_prompt_id"] + newest = self.capture( + db, 0, bundle(" fog\n", positive="a lighthouse\n"), time=100 + ) + with db.recorded_history_snapshot(): + found = db.get_newest_pair_run(positive, pair) + self.assertEqual(found["run"]["run_id"], newest.run_id) diff --git a/utils/hashing.py b/utils/hashing.py index 9bfa924e..65582f8e 100644 --- a/utils/hashing.py +++ b/utils/hashing.py @@ -65,6 +65,43 @@ def generate_prompt_hash(text: str) -> str: return hashlib.sha256(normalized_text.encode("utf-8")).hexdigest() +def stored_prompt_text(text: str) -> str: + """The text the Library stores for raw node text: outer whitespace removed. + + Every prompt writer stores ``text.strip()`` and the hash above strips too, + so this is the form recorded raw run text is compared in (#438). Case and + inner whitespace stay significant. + """ + return text.strip() + + +# The code points str.strip() removes (every str.isspace() character), for +# SQLite's char(); tests/test_hashing.py pins it to the running Python. +STRIPPED_CODE_POINTS = ( + *range(0x09, 0x0E), + *range(0x1C, 0x21), + 0x85, + 0xA0, + 0x1680, + *range(0x2000, 0x200B), + 0x2028, + 0x2029, + 0x202F, + 0x205F, + 0x3000, +) +_STRIP_CODES = ",".join(map(str, STRIPPED_CODE_POINTS)) + + +def stored_prompt_text_sql(expression: str) -> str: + """``stored_prompt_text`` as an SQLite expression over ``expression``. + + SQLite's one-argument trim() removes spaces only; trim(X, Y) removes every + character of Y, multi-byte ones included, so it strips what Python does. + """ + return f"trim({expression}, char({_STRIP_CODES}))" + + def is_duplicate_prompt(text1: str, text2: str, threshold: float = 0.95) -> bool: """ Check if two prompts are likely duplicates using hash comparison. diff --git a/web/js/workspace/history.mjs b/web/js/workspace/history.mjs index 992b895f..75c458bd 100644 --- a/web/js/workspace/history.mjs +++ b/web/js/workspace/history.mjs @@ -173,7 +173,7 @@ export function mountRecordedHistory(container, {api, positiveId, parentScope, s } count.textContent = window.group ? `${countOf(window.recorded_run_count, 'distinct recorded run')} as of this read · ${window.items.length} loaded positions` : ''; const group = window.group; - scopeText.textContent = group?.scope_status === 'uncapturable_text' ? 'This saved positive cannot be matched within frozen-text capture limits.' : 'Runs are grouped by exact frozen positive raw text, independently of saved pairs. Effective text, prepend/append composition and evidence can differ. Reported output locations do not establish context membership or historical file identity.'; + scopeText.textContent = group?.scope_status === 'uncapturable_text' ? 'This saved positive cannot be matched within frozen-text capture limits.' : 'Runs are grouped by frozen positive raw text that is this saved text apart from outer whitespace, independently of saved pairs. Effective text, prepend/append composition and evidence can differ. Reported output locations do not establish context membership or historical file identity.'; status.setAttribute('role', window.error || next.run.error ? 'alert' : 'status'); status.textContent = next.validating ? 'Revalidating recorded history…' : window.loading ? 'Loading recorded history…' : window.error ? 'Recorded history could not be loaded.' : next.refreshRequired ? 'Recorded evidence changed. Reload before using these actions.' : !items.length ? 'No matching recorded executions.' : window.has_more ? 'Newest bounded window · displayed in chronological order.' : 'All matching recorded runs in this window are loaded.'; if (next.run.error && !next.selection.run_id && !next.refreshRequired) status.textContent += ' The previous recorded selection could not be restored.';